From 4dd15aa7d7c2aae0e4d96e4ac0a4c1423afd396e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 12 Mar 2020 18:36:38 -0500 Subject: [PATCH 001/255] Fix #169 - header pragma omitted by default --- nimterop/ast.nim | 4 ++-- nimterop/getters.nim | 22 +++++++--------------- nimterop/globals.nim | 2 +- nimterop/grammar.nim | 4 ++-- nimterop/toast.nim | 13 ++++++++----- tests/include/test2.hpp | 10 ++++++---- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index a61e65a..e28241f 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -178,7 +178,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable nimState.impShort = nimState.currentHeader.replace("header", "imp") nimState.sourceFile = fullpath - if nimState.gState.dynlib.Bl: + if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: nimState.constStr &= &"\n {nimState.currentHeader} {{.used.}} = \"{fp}\"" root.searchAst(astTable, nimState) @@ -191,7 +191,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable necho &"const{nimState.constStr}\n" necho &""" -{{.pragma: {nimState.impShort}, importc{nimState.getHeader()}.}} +{{.pragma: {nimState.impShort}, importc{nimState.getHeaderPragma()}.}} {{.pragma: {nimState.impShort}C, {nimState.impShort}, cdecl{nimState.getDynlib()}.}} """ diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 4cd8c04..93c4000 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -346,8 +346,7 @@ proc printLisp*(gState: State, root: TSNode): string = while true: if not node.isNil() and depth > -1: - if gState.pretty: - result &= spaces(depth) + result &= spaces(depth) let (line, col) = gState.getLineCol(node) result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" @@ -359,15 +358,11 @@ proc printLisp*(gState: State, root: TSNode): string = break if node.tsNodeNamedChildCount() != 0: - if gState.pretty: - result &= "\n" + result &= "\n" nextnode = node.tsNodeNamedChild(0) depth += 1 else: - if gState.pretty: - result &= ")\n" - else: - result &= ")" + result &= ")\n" nextnode = node.tsNodeNextNamedSibling() if nextnode.isNil(): @@ -376,10 +371,7 @@ proc printLisp*(gState: State, root: TSNode): string = depth -= 1 if depth == -1: break - if gState.pretty: - result &= spaces(depth) & ")\n" - else: - result &= ")" + result &= spaces(depth) & ")\n" if node == root: break if not node.tsNodeNextNamedSibling().isNil(): @@ -656,9 +648,9 @@ proc getSplitComma*(joined: seq[string]): seq[string] = for i in joined: result = result.concat(i.split(",")) -proc getHeader*(nimState: NimState): string = +proc getHeaderPragma*(nimState: NimState): string = result = - if nimState.gState.dynlib.Bl: + if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: &", header: {nimState.currentHeader}" else: "" @@ -672,7 +664,7 @@ proc getDynlib*(nimState: NimState): string = proc getImportC*(nimState: NimState, origName, nimName: string): string = if nimName != origName: - result = &"importc: \"{origName}\"{nimState.getHeader()}" + result = &"importc: \"{origName}\"{nimState.getHeaderPragma()}" else: result = nimState.impShort diff --git a/nimterop/globals.nim b/nimterop/globals.nim index b234c1f..35a5575 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -57,7 +57,7 @@ type State = ref object compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string] - nocache*, nocomments*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool + debug*, includeHeader*, nocache*, nocomments*, past*, preprocess*, pnim*, recurse*: bool code*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index c76844c..f15c2f8 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -200,7 +200,7 @@ proc initGrammar(): Grammar = nname = nimState.getIdentifier(name, nskType) i += 1 - if nimState.gState.dynlib.Bl: + if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: pragmas.add nimState.getImportC(name, nname) let @@ -316,7 +316,7 @@ proc initGrammar(): Grammar = else: var pragmas: seq[string] = @[] - if nimState.gState.dynlib.Bl: + if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: pragmas.add nimState.getImportC(prefix & name, nname) pragmas.add "bycopy" if union.nBl: diff --git a/nimterop/toast.nim b/nimterop/toast.nim index a4415d2..323b4ec 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -58,6 +58,7 @@ proc main( defines: seq[string] = @[], dynlib: string = "", feature: seq[Feature] = @[], + includeHeader = false, includeDirs: seq[string] = @[], mode = modeDefault, nim: string = "nim", @@ -82,6 +83,7 @@ proc main( defines: defines, dynlib: dynlib, feature: feature, + includeHeader: includeHeader, includeDirs: includeDirs, mode: mode, nim: nim, @@ -91,7 +93,6 @@ proc main( pnim: pnim, prefix: prefix, preprocess: preprocess, - pretty: true, recurse: recurse, suffix: suffix, symOverride: symOverride @@ -190,8 +191,9 @@ when isMainModule: "check": "check generated wrapper with compiler", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", - "dynlib": "Import symbols from library in specified Nim string", + "dynlib": "import symbols from library in specified Nim string", "feature": "flags to enable experimental features", + "includeHeader": "add {.header.} pragma to wrapper", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", "nim": "use a particular Nim executable (default: $PATH/nim)", @@ -199,14 +201,14 @@ when isMainModule: "output": "file to output content - default stdout", "past": "print AST output", "pgrammar": "print grammar", - "pluginSourcePath": "Nim file to build and load as a plugin", + "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", "preprocess": "run preprocessor on header", "recurse": "process #include files", "source" : "C/C++ source/header", - "prefix": "Strip prefix from identifiers", + "prefix": "strip prefix from identifiers", "stub": "stub out undefined type references as objects", - "suffix": "Strip suffix from identifiers", + "suffix": "strip suffix from identifiers", "symOverride": "skip generating specified symbols" }, short = { "check": 'k', @@ -214,6 +216,7 @@ when isMainModule: "defines": 'D', "dynlib": 'l', "feature": 'f', + "includeHeader": 'H', "includeDirs": 'I', "nocomments": 'c', "output": 'o', diff --git a/tests/include/test2.hpp b/tests/include/test2.hpp index 0fd90ee..2f9a281 100644 --- a/tests/include/test2.hpp +++ b/tests/include/test2.hpp @@ -4,11 +4,13 @@ #define TEST_FLOAT 5.12 #define TEST_HEX 0x512 -int test_call_int(); +extern "C" { + int test_call_int(); -struct Foo{ - int bar; -}; + struct Foo{ + int bar; + }; +} class Foo1{ int bar1; From 077981c3a5e577029cf28bffc67648e73d4aafcc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 15 Mar 2020 22:23:22 -0500 Subject: [PATCH 002/255] Readme update --- README.md | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ca16293..b1fee44 100644 --- a/README.md +++ b/README.md @@ -90,31 +90,33 @@ Refer to the ```tests``` directory for examples on how the library can be used. The `toast` binary can also be used directly on the CLI: ``` -toast -h +> toast -h Usage: main [optional-params] C/C++ source/header -Options(opt-arg sep :|=|spc): - -h, --help print this cligen-erated help - --help-syntax advanced: prepend,plurals,.. - -k, --check bool false check generated wrapper with compiler - -d, --debug bool false enable debug output - -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" Import symbols from library in specified Nim string - -I=, --includeDirs= strings {} include directory to pass to preprocessor - -m=, --mode= string "cpp" language parser: c or cpp - --nim= string "nim" use a particular Nim executable (default: $PATH/nim) - -c, --nocomments bool false exclude top-level comments from output - -o=, --output= string "" file to output content - default stdout - -a, --past bool false print AST output - -g, --pgrammar bool false print grammar - --pluginSourcePath= string "" Nim file to build and load as a plugin - -n, --pnim bool false print Nim output - -E=, --prefix= strings {} Strip prefix from identifiers - -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files - -s, --stub bool false stub out undefined type references as objects - -F=, --suffix= strings {} Strip suffix from identifiers - -O=, --symOverride= strings {} skip generating specified symbols +Options: + -h, --help print this cligen-erated help + --help-syntax advanced: prepend,plurals,.. + -k, --check bool false check generated wrapper with compiler + -d, --debug bool false enable debug output + -D=, --defines= strings {} definitions to pass to preprocessor + -l=, --dynlib= string "" import symbols from library in specified Nim string + -f=, --feature= Features {} flags to enable experimental features + -H, --includeHeader bool false add {.header.} pragma to wrapper + -I=, --includeDirs= strings {} include directory to pass to preprocessor + -m=, --mode= string "cpp" language parser: c or cpp + --nim= string "nim" use a particular Nim executable (default: $PATH/nim) + -c, --nocomments bool false exclude top-level comments from output + -o=, --output= string "" file to output content - default stdout + -a, --past bool false print AST output + -g, --pgrammar bool false print grammar + --pluginSourcePath= string "" nim file to build and load as a plugin + -n, --pnim bool false print Nim output + -E=, --prefix= strings {} strip prefix from identifiers + -p, --preprocess bool false run preprocessor on header + -r, --recurse bool false process #include files + -s, --stub bool false stub out undefined type references as objects + -F=, --suffix= strings {} strip suffix from identifiers + -O=, --symOverride= strings {} skip generating specified symbols ``` __Implementation Details__ From 3f500898aa5b45b423348846d794463585ef34ee Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 16 Mar 2020 17:50:43 -0500 Subject: [PATCH 003/255] Add ast2 array expression support, cleanup debug output --- nimterop/ast2.nim | 24 +++++++++++++++--------- nimterop/getters.nim | 30 +++++++++++++++--------------- nimterop/globals.nim | 3 ++- tests/include/tast2.h | 2 +- tests/tast2.nim | 3 +++ 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 697262f..f418139 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -2,7 +2,7 @@ import macros, os, sets, strutils, tables, times import regex -import compiler/[ast, idents, options, renderer] +import compiler/[ast, idents, modulegraphs, options, parser, renderer] import "."/treesitter/api @@ -19,8 +19,9 @@ proc getPtrType*(str: string): string = else: str -proc getLit*(str: string): PNode = - # Used to convert #define literals into const +proc getLit*(nimState: NimState, str: string): PNode = + # Used to convert #define literals into const and expressions + # in array sizes let str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() @@ -30,9 +31,9 @@ proc getLit*(str: string): PNode = elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float result = newFloatNode(nkFloatLit, parseFloat(str)) - # TODO - hex becomes int on render - elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - result = newIntNode(nkIntLit, parseHexInt(str)) + # # TODO - hex becomes int on render + # elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + # result = newIntNode(nkIntLit, parseHexInt(str)) elif str.contains(re"^'[[:ascii:]]'$"): # char result = newNode(nkCharLit) @@ -42,7 +43,11 @@ proc getLit*(str: string): PNode = result = newStrNode(nkStrLit, str[1 .. ^2]) else: - result = newNode(nkNilLit) + result = parseString( + nimState.getNimExpression(str), + nimState.identCache, nimState.config) + if result.isNil: + result = newNode(nkNilLit) proc addConst(nimState: NimState, node: TSNode) = # #define X Y @@ -65,7 +70,7 @@ proc addConst(nimState: NimState, node: TSNode) = ident = nimState.getIdent(name, info) # node[1] = preproc_arg = value - val = nimState.getNodeVal(node[1]).getLit() + val = nimState.getLit(nimState.getNodeVal(node[1])) # If supported literal if val.kind != nkNilLit: @@ -464,7 +469,7 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = for i in 0 ..< acount: let - size = nimState.getNodeVal(cnode[1]).getLit() + size = nimState.getLit(nimState.getNodeVal(cnode[1])) if size.kind != nkNilLit: result = nimState.newArrayTree(cnode, result, size) cnode = cnode[0] @@ -877,6 +882,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = # Nim compiler objects nimState.identCache = newIdentCache() nimState.config = newConfigRef() + nimstate.graph = newModuleGraph(nimState.identCache, nimState.config) nimState.constSection = newNode(nkConstSection) nimState.enumSection = newNode(nkStmtList) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 93c4000..2c7da75 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -386,33 +386,33 @@ proc printLisp*(gState: State, root: TSNode): string = proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") -proc printTree*(nimState: NimState, pnode: PNode, offset = "") = +proc printTree*(nimState: NimState, pnode: PNode, offset = ""): string = if nimState.gState.debug and pnode.kind != nkNone: - stdout.write "\n# " & offset & $pnode.kind & "(" + result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: - stdout.write "'" & pnode.intVal.char & "')" + result &= "'" & pnode.intVal.char & "')" of nkIntLit..nkUInt64Lit: - stdout.write $pnode.intVal & ")" + result &= $pnode.intVal & ")" of nkFloatLit..nkFloat128Lit: - stdout.write $pnode.floatVal & ")" + result &= $pnode.floatVal & ")" of nkStrLit..nkTripleStrLit: - stdout.write "\"" & pnode.strVal & "\")" + result &= "\"" & pnode.strVal & "\")" of nkSym: - stdout.write $pnode.sym & ")" + result &= $pnode.sym & ")" of nkIdent: - stdout.write "\"" & $pnode.ident.s & "\")" + result &= "\"" & $pnode.ident.s & "\")" else: if pnode.sons.len != 0: for i in 0 ..< pnode.sons.len: - nimState.printTree(pnode.sons[i], offset & " ") + result &= nimState.printTree(pnode.sons[i], offset & " ") if i != pnode.sons.len - 1: - stdout.write "," - stdout.write "\n# " & offset & ")" + result &= "," + result &= "\n# " & offset & ")" else: - stdout.write ")" + result &= ")" if offset.len == 0: - necho "" + result &= "\n" proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: @@ -421,8 +421,8 @@ proc printDebug*(nimState: NimState, node: TSNode) = proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: - necho ("Output => " & $pnode).getCommented() - nimState.printTree(pnode) + necho ("Output => " & $pnode).getCommented() & "\n" & + nimState.printTree(pnode) # Compiler shortcuts diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 35a5575..ba63a76 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -5,7 +5,7 @@ import regex import "."/plugin when not declared(CIMPORT): - import compiler/[ast, idents, options] + import compiler/[ast, idents, modulegraphs, options] import "."/treesitter/api @@ -81,6 +81,7 @@ type constSection*, enumSection*, procSection*, typeSection*: PNode identCache*: IdentCache config*: ConfigRef + graph*: ModuleGraph gState*: State diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 1a2789b..ced3a44 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -32,7 +32,7 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p; typedef struct A20 { char a1; } A20, A21, *A21p; //Expression -//typedef struct A21 { int **f1; int abc[123+132]; } A21; +typedef struct A22 { int **f1; int *f2[123+132]; } A22; //Unions //union UNION1 {int f1; }; diff --git a/tests/tast2.nim b/tests/tast2.nim index ffdcd19..dd8fae7 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -69,3 +69,6 @@ assert A20 is object testFields(A20, {"a1": "cchar"}.toTable()) assert A21 is A20 assert A21p is ptr A20 + +assert A22 is object +testFields(A22, {"f1": "ptr ptr cint", "f2": "array[0..254, ptr cint]"}.toTable()) \ No newline at end of file From 1a9911fc259a5dd5ac9a6097c4d41d6e3e50169b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 16 Mar 2020 23:15:34 -0500 Subject: [PATCH 004/255] Add ast2 union support --- nimterop/ast2.nim | 77 +++++++++++++++++++++++++++++++++++++------ tests/include/tast2.h | 4 +-- tests/tast2.nim | 8 ++++- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index f418139..ca0fc51 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -93,7 +93,55 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) -proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = +proc newPragma(nimState: NimState, node: TSNode, pragmas: Table[string, string]): PNode = + # Create nkPragma tree for name:value + # + # {.name1, name2: value2.} + # + # nkPragma( + # nkIdent(name1), + # nkExprColonExpr( + # nkIdent(name2), + # nkStrLit(value2) + # ) + # ) + result = newNode(nkPragma) + for name, value in pragmas.pairs: + let + (_, pinfo) = nimState.getNameInfo(node, nskUnknown) + pident = nimState.getIdent(name, pinfo, exported = false) + + if value.len == 0: + result.add pident + else: + let + colExpr = newNode(nkExprColonExpr) + pvalue = newStrNode(nkStrLit, value) + colExpr.add pident + colExpr.add pvalue + result.add colExpr + +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: Table[string, string]): PNode = + # Create nkPragmaExpr tree + # + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkPragma( + # nkIdent(name1), + # nkExprColonExpr( + # nkIdent(name2), + # nkStrLit(value2) + # ) + # ) + # ) + result = newNode(nkPragmaExpr) + result.add ident + result.add nimState.newPragma(node, pragmas) + +proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false): PNode = # Create nkTypeDef PNode with first ident # # If `override`, use it instead of node.getAtom() for name @@ -105,6 +153,11 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = nimState.getIdent(override, info) else: nimState.getIdent(name, info) + prident = + if union: + nimState.newPragmaExpr(node, ident, {"union": ""}.toTable()) + else: + ident # type name* = # @@ -116,7 +169,7 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = # nkEmpty() # ) result = newNode(nkTypeDef) - result.add ident + result.add prident result.add newNode(nkEmpty) proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = @@ -325,7 +378,7 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = if not field.isNil: result.add field -proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "") = +proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "", union = false) = # Add a type of object # # If `override` is set, use it as the name @@ -333,7 +386,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = decho("addTypeObject()") let # TODO - check blank and override - typeDef = nimState.newTypeIdent(node, override) + typeDef = nimState.newTypeIdent(node, override, union = union) name = $typeDef[0][1] if name != duplicate: @@ -597,11 +650,11 @@ proc addTypeProc(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc addType(nimState: NimState, node: TSNode) = +proc addType(nimState: NimState, node: TSNode, union = false) = decho("addType()") nimState.printDebug(node) - if node.getName() == "struct_specifier": + if node.getName() in ["struct_specifier", "union_specifier"]: # struct X; # # (struct_specifier @@ -631,7 +684,7 @@ proc addType(nimState: NimState, node: TSNode) = # (field_declaration ...) # ) decho("addType(): case 1") - nimState.addTypeObject(node) + nimState.addTypeObject(node, union = union) elif node.getName() == "type_definition": if node.len >= 2: let @@ -657,7 +710,7 @@ proc addType(nimState: NimState, node: TSNode) = # (type_definition = "") # ) decho("addType(): case 2") - nimState.addTypeObject(node[0]) + nimState.addTypeObject(node[0], union = union) else: let fdecl = node[1].anyChildInTree("function_declarator") @@ -763,7 +816,7 @@ proc addType(nimState: NimState, node: TSNode) = # First add struct as object decho("addType(): case 6") - nimState.addTypeObject(node[0]) + nimState.addTypeObject(node[0], union = union) if node.len > 1 and nimState.getNodeVal(node[1]) != "": # Add any additional names @@ -786,7 +839,7 @@ proc addType(nimState: NimState, node: TSNode) = name # Now add struct as object with specified name - nimState.addTypeObject(node[0], override = name) + nimState.addTypeObject(node[0], override = name, union = union) if name.len != 0: # Add any additional names except duplicate @@ -809,10 +862,14 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "type_definition": if not node.firstChildInTree("enum_specifier").isNil(): nimState.addEnum(node) + elif not node.firstChildInTree("union_specifier").isNil(): + nimState.addType(node, union = true) else: nimState.addType(node) of "struct_specifier": nimState.addType(node) + of "union_specifier": + nimState.addType(node, union = true) of "enum_specifier": nimState.addEnum(node) of "declaration": diff --git a/tests/include/tast2.h b/tests/include/tast2.h index ced3a44..90e96c6 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,8 +35,8 @@ typedef struct A20 { char a1; } A20, A21, *A21p; typedef struct A22 { int **f1; int *f2[123+132]; } A22; //Unions -//union UNION1 {int f1; }; -//typedef union UNION2 { int **f1; int abc[123+132]; } UNION2; +union U1 {int f1; float f2; }; +typedef union U2 { int **f1; int abc[123+132]; } U2; // Anonymous //typedef struct { char a1; }; diff --git a/tests/tast2.nim b/tests/tast2.nim index dd8fae7..d8d18a3 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -71,4 +71,10 @@ assert A21 is A20 assert A21p is ptr A20 assert A22 is object -testFields(A22, {"f1": "ptr ptr cint", "f2": "array[0..254, ptr cint]"}.toTable()) \ No newline at end of file +testFields(A22, {"f1": "ptr ptr cint", "f2": "array[0..254, ptr cint]"}.toTable()) + +assert U1 is object +assert sizeof(U1) == sizeof(cfloat) + +assert U2 is object +assert sizeof(U2) == 256 * sizeof(cint) \ No newline at end of file From 9d2626bb5fd1603ed0f6db1757b0184d9e5b29e4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 17 Mar 2020 16:34:11 -0500 Subject: [PATCH 005/255] Initial ast2 proc support --- nimterop/ast2.nim | 121 +++++++++++++++++++++++++++++++++++------- tests/include/tast2.h | 2 + 2 files changed, 103 insertions(+), 20 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index ca0fc51..8c2fc3f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,4 +1,4 @@ -import macros, os, sets, strutils, tables, times +import macros, os, sets, strformat, strutils, tables, times import regex @@ -19,6 +19,9 @@ proc getPtrType*(str: string): string = else: str +proc parseString(nimState: NimState, str: string): PNode = + result = parseString(str, nimState.identCache, nimState.config) + proc getLit*(nimState: NimState, str: string): PNode = # Used to convert #define literals into const and expressions # in array sizes @@ -43,9 +46,7 @@ proc getLit*(nimState: NimState, str: string): PNode = result = newStrNode(nkStrLit, str[1 .. ^2]) else: - result = parseString( - nimState.getNimExpression(str), - nimState.identCache, nimState.config) + result = nimState.parseString(nimState.getNimExpression(str)) if result.isNil: result = newNode(nkNilLit) @@ -93,7 +94,7 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) -proc newPragma(nimState: NimState, node: TSNode, pragmas: Table[string, string]): PNode = +proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, string]): PNode = # Create nkPragma tree for name:value # # {.name1, name2: value2.} @@ -108,7 +109,7 @@ proc newPragma(nimState: NimState, node: TSNode, pragmas: Table[string, string]) result = newNode(nkPragma) for name, value in pragmas.pairs: let - (_, pinfo) = nimState.getNameInfo(node, nskUnknown) + (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) pident = nimState.getIdent(name, pinfo, exported = false) if value.len == 0: @@ -121,7 +122,7 @@ proc newPragma(nimState: NimState, node: TSNode, pragmas: Table[string, string]) colExpr.add pvalue result.add colExpr -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: Table[string, string]): PNode = +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, string]): PNode = # Create nkPragmaExpr tree # # nkPragmaExpr( @@ -155,7 +156,7 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false nimState.getIdent(name, info) prident = if union: - nimState.newPragmaExpr(node, ident, {"union": ""}.toTable()) + nimState.newPragmaExpr(node, ident, {"union": ""}.toOrderedTable()) else: ident @@ -215,7 +216,7 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # Create nkBracketExpr tree depending on input let - (_, info) = nimState.getNameInfo(node, nskType) + (_, info) = nimState.getNameInfo(node.getAtom(), nskType) ident = nimState.getIdent("array", info, exported = false) # array[size, typ] @@ -264,7 +265,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn start = getStartAtom(node) # node[start] - param type - (tname, tinfo) = nimState.getNameInfo(node[start], nskType) + (tname, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) tident = nimState.getIdent(tname, tinfo, exported = false) if start == node.len - 1: @@ -327,13 +328,21 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn else: result = nil -proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = - # Create nkProcTy tree for specified proc type - let - fparam = newNode(nkFormalParams) +proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + # Create nkFormalParams tree for specified params and return type + # + # proc(pname: ptyp ..): rtyp + # + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ) + result = newNode(nkFormalParams) # Add return type - fparam.add rtyp + result.add rtyp if not node.isNil: for i in 0 ..< node.len: @@ -341,7 +350,10 @@ proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): P let param = nimState.newIdentDefs(name, node[i], i, exported = false) if not param.isNil: - fparam.add param + result.add param + +proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + # Create nkProcTy tree for specified proc type # proc(pname: ptyp ..): rtyp # @@ -355,7 +367,7 @@ proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): P # nkEmpty() # ) result = newNode(nkProcTy) - result.add fparam + result.add nimState.newFormalParams(name, node, rtyp) result.add newNode(nkEmpty) proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = @@ -600,7 +612,7 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = retType = nimState.newPtrTree(tcount, retType) # Proc with return type and params - result = nimState.newProcTree(name, plist, retType) + result = nimState.newProcTy(name, plist, retType) if ncount > 1: result = nimState.newPtrTree(ncount-1, result) @@ -850,9 +862,77 @@ proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) proc addProc(nimState: NimState, node: TSNode) = + # Add a proc decho("addProc()") nimState.printDebug(node) + let + start = getStartAtom(node) + + # node[start] = identifier = return type name + (rname, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) + + # Parameter list + plist = node[start+1].anyChildInTree("parameter_list") + + # node[start+1] = identifier = name + # TODO - check blank and override + ident = nimState.newTypeIdent(node[start+1]) + name = $ident[0][1] + + # node[start+1] could have nested pointers + tcount = node[start+1].getPtrCount() + + procDef = newNode(nkProcDef) + + # proc X(a1: Y, a2: Z): P {.pragma.} + # + # nkProcDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkEmpty(), + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkPragma(...), + # nkEmpty(), + # nkEmpty() + # ) + + procDef.add ident + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) + + # Return type + var + retType = nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = nimState.newPtrTree(tcount, retType) + + # Proc with return type and params + procDef.add nimState.newFormalParams(name, plist, retType) + procDef.add newNode(nkEmpty) # Pragmas + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) + + # nkProcSection.add + nimState.procSection.add procDef + + nimState.printDebug(procDef) + proc processNode(nimState: NimState, node: TSNode): bool = result = true @@ -926,14 +1006,15 @@ import nimterop/types """ % [$now(), getAppFilename(), commandLineParams().join(" ")] proc printNim*(gState: State, fullpath: string, root: TSNode) = - var + let nimState = new(NimState) - #fp = fullpath.replace("\\", "/") + fp = fullpath.replace("\\", "/") nimState.identifiers = newTable[string, string]() nimState.gState = gState nimState.currentHeader = getCurrentHeader(fullpath) + nimState.impShort = nimState.currentHeader.replace("header", "imp") nimState.sourceFile = fullpath # Nim compiler objects diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 90e96c6..200b4c8 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -40,3 +40,5 @@ typedef union U2 { int **f1; int abc[123+132]; } U2; // Anonymous //typedef struct { char a1; }; + +struct A2 test_proc1(struct A0 a); \ No newline at end of file From 795f607e96a3228b9daa6dd6384fa63dab35c14e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 17 Mar 2020 17:19:47 -0500 Subject: [PATCH 006/255] newConstDef for header --- nimterop/ast2.nim | 67 +++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8c2fc3f..536ef9b 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -50,6 +50,42 @@ proc getLit*(nimState: NimState, str: string): PNode = if result.isNil: result = newNode(nkNilLit) +proc newConstDef(nimState: NimState, node: TSNode, name = "", val = ""): PNode = + let + # node[0] = identifier = const name + (cname, info) = nimState.getNameInfo(node.getAtom(), nskConst) + + # TODO - check blank and override + ident = + if name.len != 0: + nimState.getIdent(name, info) + else: + nimState.getIdent(cname, info) + + # node[1] = preproc_arg = value + nval = + if val.len != 0: + newStrNode(nkStrLit, val) + else: + nimState.getLit(nimState.getNodeVal(node[1])) + + # If supported literal + if nval.kind != nkNilLit: + # const X* = Y + # + # nkConstDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkXLit(Y) + # ) + result = newNode(nkConstDef) + result.add ident + result.add newNode(nkEmpty) + result.add nval + proc addConst(nimState: NimState, node: TSNode) = # #define X Y # @@ -63,32 +99,9 @@ proc addConst(nimState: NimState, node: TSNode) = if node[0].getName() == "identifier" and node[1].getName() == "preproc_arg": let - constDef = newNode(nkConstDef) - - # node[0] = identifier = const name - (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) - # TODO - check blank and override - ident = nimState.getIdent(name, info) - - # node[1] = preproc_arg = value - val = nimState.getLit(nimState.getNodeVal(node[1])) - - # If supported literal - if val.kind != nkNilLit: - # const X* = Y - # - # nkConstDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkXLit(Y) - # ) - constDef.add ident - constDef.add newNode(nkEmpty) - constDef.add val + constDef = nimState.newConstDef(node) + if not constDef.isNil: # nkConstSection.add nimState.constSection.add constDef @@ -1027,6 +1040,10 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = nimState.procSection = newNode(nkStmtList) nimState.typeSection = newNode(nkTypeSection) + if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: + nimState.constSection.add nimState.newConstDef( + root, name = nimState.currentHeader, val = fp) + nimState.searchTree(root) var From 2d45c6adf609994046396f8e574fab8534bcfa6e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 17 Mar 2020 23:43:47 -0500 Subject: [PATCH 007/255] Initial ast2 pragma support --- nimterop/ast.nim | 38 ++++++++++++++++++++- nimterop/ast2.nim | 78 ++++++++++++++++++++++++++++++++------------ nimterop/getters.nim | 43 +++++------------------- nimterop/globals.nim | 2 +- nimterop/grammar.nim | 6 ++-- 5 files changed, 107 insertions(+), 60 deletions(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index e28241f..06b0504 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -4,6 +4,42 @@ import regex import "."/[getters, globals, treesitter/api] +proc getHeaderPragma*(nimState: NimState): string = + result = + if nimState.includeHeader(): + &", header: {nimState.currentHeader}" + else: + "" + +proc getDynlib*(nimState: NimState): string = + result = + if nimState.gState.dynlib.nBl: + &", dynlib: {nimState.gState.dynlib}" + else: + "" + +proc getImportC*(nimState: NimState, origName, nimName: string): string = + if nimName != origName: + result = &"importc: \"{origName}\"{nimState.getHeaderPragma()}" + else: + result = nimState.impShort + +proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = + result = "" + for pragma in pragmas.items(): + if pragma.nBl: + result &= pragma & ", " + if result.nBl: + result = " {." & result[0 .. ^3] & ".}" + + result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C") + + let + dy = nimState.getDynlib() + + if ", cdecl" in result and dy.nBl: + result = result.replace(".}", dy & ".}") + proc saveNodeData(node: TSNode, nimState: NimState): bool = let name = $node.tsNodeType() @@ -178,7 +214,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable nimState.impShort = nimState.currentHeader.replace("header", "imp") nimState.sourceFile = fullpath - if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: + if nimState.includeHeader(): nimState.constStr &= &"\n {nimState.currentHeader} {{.used.}} = \"{fp}\"" root.searchAst(astTable, nimState) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 536ef9b..8804c53 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -107,7 +107,23 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) -proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, string]): PNode = +proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: OrderedTable[string, PNode]) = + # Add pragmas to an existing nkPragma tree + for name, value in pragmas.pairs: + let + (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) + pident = nimState.getIdent(name, pinfo, exported = false) + + if value.isNil: + pragma.add pident + else: + let + colExpr = newNode(nkExprColonExpr) + colExpr.add pident + colExpr.add value + pragma.add colExpr + +proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, PNode]): PNode = # Create nkPragma tree for name:value # # {.name1, name2: value2.} @@ -120,22 +136,9 @@ proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, s # ) # ) result = newNode(nkPragma) - for name, value in pragmas.pairs: - let - (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) - pident = nimState.getIdent(name, pinfo, exported = false) - - if value.len == 0: - result.add pident - else: - let - colExpr = newNode(nkExprColonExpr) - pvalue = newStrNode(nkStrLit, value) - colExpr.add pident - colExpr.add pvalue - result.add colExpr + nimState.addPragma(node, result, pragmas) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, string]): PNode = +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode = # Create nkPragmaExpr tree # # nkPragmaExpr( @@ -169,7 +172,9 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false nimState.getIdent(name, info) prident = if union: - nimState.newPragmaExpr(node, ident, {"union": ""}.toOrderedTable()) + var + empty: PNode + nimState.newPragmaExpr(node, ident, {"union": empty}.toOrderedTable()) else: ident @@ -1008,6 +1013,39 @@ proc searchTree(nimState: NimState, root: TSNode) = if node == root: break +proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = + var + impPragma = newNode(nkPragma) + impCPragma = newNode(nkPragma) + empty: PNode + + nimState.addPragma(root, impPragma, { + "pragma": nimState.getIdent(nimState.impShort), + "importc": empty + }.toOrderedTable()) + + if nimState.includeHeader(): + nimState.constSection.add nimState.newConstDef( + root, name = nimState.currentHeader, val = fullpath) + + nimState.addPragma(root, impPragma, { + "header": newStrNode(nkStrLit, nimState.currentHeader) + }.toOrderedTable()) + + nimState.addPragma(root, impCPragma, { + "pragma": nimState.getIdent(nimState.impShort & "C"), + nimState.impShort: empty, + "cdecl": empty + }.toOrderedTable()) + + if nimState.gState.dynlib.nBl: + nimState.addPragma(root, impCPragma, { + "dynlib": nimState.getIdent(nimState.gState.dynlib) + }.toOrderedTable()) + + nimState.pragmaSection.add impPragma + nimState.pragmaSection.add impCPragma + proc printNimHeader*(gState: State) = gecho """# Generated at $1 # Command line: @@ -1035,19 +1073,19 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = nimState.config = newConfigRef() nimstate.graph = newModuleGraph(nimState.identCache, nimState.config) + nimState.pragmaSection = newNode(nkStmtList) nimState.constSection = newNode(nkConstSection) nimState.enumSection = newNode(nkStmtList) nimState.procSection = newNode(nkStmtList) nimState.typeSection = newNode(nkTypeSection) - if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: - nimState.constSection.add nimState.newConstDef( - root, name = nimState.currentHeader, val = fp) + nimState.setupPragmas(root, fp) nimState.searchTree(root) var tree = newNode(nkStmtList) + tree.add nimState.pragmaSection tree.add nimState.enumSection tree.add nimState.constSection tree.add nimState.typeSection diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 2c7da75..07ed194 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -426,6 +426,9 @@ proc printDebug*(nimState: NimState, pnode: PNode) = # Compiler shortcuts +proc getDefaultLineInfo*(nimState: NimState): TLineInfo = + result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, -1, -1) + proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = # Get Nim equivalent line:col info from node let @@ -446,6 +449,9 @@ proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = tru else: result = newIdentNode(ident, info) +proc getIdent*(nimState: NimState, name: string): PNode = + nimState.getIdent(name, nimState.getDefaultLineInfo(), exported = false) + proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): tuple[name: string, info: TLineInfo] = # Shortcut to get identifier name and info (node value and line:col) @@ -648,41 +654,8 @@ proc getSplitComma*(joined: seq[string]): seq[string] = for i in joined: result = result.concat(i.split(",")) -proc getHeaderPragma*(nimState: NimState): string = - result = - if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: - &", header: {nimState.currentHeader}" - else: - "" - -proc getDynlib*(nimState: NimState): string = - result = - if nimState.gState.dynlib.nBl: - &", dynlib: {nimState.gState.dynlib}" - else: - "" - -proc getImportC*(nimState: NimState, origName, nimName: string): string = - if nimName != origName: - result = &"importc: \"{origName}\"{nimState.getHeaderPragma()}" - else: - result = nimState.impShort - -proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = - result = "" - for pragma in pragmas.items(): - if pragma.nBl: - result &= pragma & ", " - if result.nBl: - result = " {." & result[0 .. ^3] & ".}" - - result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C") - - let - dy = nimState.getDynlib() - - if ", cdecl" in result and dy.nBl: - result = result.replace(".}", dy & ".}") +template includeHeader*(nimState: NimState): bool = + nimState.gState.dynlib.Bl and nimState.gState.includeHeader proc getComments*(nimState: NimState, strip = false): string = if not nimState.gState.nocomments and nimState.commentStr.nBl: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index ba63a76..2fb543b 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -78,7 +78,7 @@ type # Nim compiler objects when not declared(CIMPORT): - constSection*, enumSection*, procSection*, typeSection*: PNode + pragmaSection*, constSection*, enumSection*, procSection*, typeSection*: PNode identCache*: IdentCache config*: ConfigRef graph*: ModuleGraph diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index f15c2f8..2e6b585 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -2,7 +2,7 @@ import macros, strformat, strutils, tables import regex -import "."/[getters, globals, lisp, treesitter/api] +import "."/[ast, getters, globals, lisp, treesitter/api] type Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, nimState: NimState) {.nimcall.}]] @@ -200,7 +200,7 @@ proc initGrammar(): Grammar = nname = nimState.getIdentifier(name, nskType) i += 1 - if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: + if nimState.includeHeader(): pragmas.add nimState.getImportC(name, nname) let @@ -316,7 +316,7 @@ proc initGrammar(): Grammar = else: var pragmas: seq[string] = @[] - if nimState.gState.dynlib.Bl and nimState.gState.includeHeader: + if nimState.includeHeader(): pragmas.add nimState.getImportC(prefix & name, nname) pragmas.add "bycopy" if union.nBl: From 8fadeeb76c9101fa7333e684c1e65d19b4d03f99 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 18 Mar 2020 13:34:14 -0500 Subject: [PATCH 008/255] Clean up ast2 pragma procs --- nimterop/ast2.nim | 72 +++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8804c53..4539f79 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -107,23 +107,27 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) +proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, value: PNode = nil) = + # Add pragma to an existing nkPragma tree + let + (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) + pident = nimState.getIdent(name, pinfo, exported = false) + + if value.isNil: + pragma.add pident + else: + let + colExpr = newNode(nkExprColonExpr) + colExpr.add pident + colExpr.add value + pragma.add colExpr + proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: OrderedTable[string, PNode]) = # Add pragmas to an existing nkPragma tree for name, value in pragmas.pairs: - let - (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) - pident = nimState.getIdent(name, pinfo, exported = false) - - if value.isNil: - pragma.add pident - else: - let - colExpr = newNode(nkExprColonExpr) - colExpr.add pident - colExpr.add value - pragma.add colExpr + nimState.addPragma(node, pragma, name, value) -proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, PNode]): PNode = +proc newPragma(nimState: NimState, node: TSNode, name: string, value: PNode = nil): PNode = # Create nkPragma tree for name:value # # {.name1, name2: value2.} @@ -136,10 +140,15 @@ proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, P # ) # ) result = newNode(nkPragma) + nimState.addPragma(node, result, name, value) + +proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, PNode]): PNode = + # Create nkPragma tree for multiple name:value + result = newNode(nkPragma) nimState.addPragma(node, result, pragmas) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode = - # Create nkPragmaExpr tree +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, name: string, value: PNode = nil): PNode = + # Create nkPragmaExpr tree for name:value # # nkPragmaExpr( # nkPostfix( @@ -156,6 +165,12 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: Orde # ) result = newNode(nkPragmaExpr) result.add ident + result.add nimState.newPragma(node, name, value) + +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode = + # Create nkPragmaExpr tree for multiple name:value + result = newNode(nkPragmaExpr) + result.add ident result.add nimState.newPragma(node, pragmas) proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false): PNode = @@ -172,9 +187,7 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false nimState.getIdent(name, info) prident = if union: - var - empty: PNode - nimState.newPragmaExpr(node, ident, {"union": empty}.toOrderedTable()) + nimState.newPragmaExpr(node, ident, "union") else: ident @@ -1017,31 +1030,22 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = var impPragma = newNode(nkPragma) impCPragma = newNode(nkPragma) - empty: PNode - nimState.addPragma(root, impPragma, { - "pragma": nimState.getIdent(nimState.impShort), - "importc": empty - }.toOrderedTable()) + nimState.addPragma(root, impPragma, "pragma", nimState.getIdent(nimState.impShort)) + nimState.addPragma(root, impPragma, "importc") if nimState.includeHeader(): nimState.constSection.add nimState.newConstDef( root, name = nimState.currentHeader, val = fullpath) - nimState.addPragma(root, impPragma, { - "header": newStrNode(nkStrLit, nimState.currentHeader) - }.toOrderedTable()) + nimState.addPragma(root, impPragma, "header", newStrNode(nkStrLit, nimState.currentHeader)) - nimState.addPragma(root, impCPragma, { - "pragma": nimState.getIdent(nimState.impShort & "C"), - nimState.impShort: empty, - "cdecl": empty - }.toOrderedTable()) + nimState.addPragma(root, impCPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) + nimState.addPragma(root, impCPragma, nimState.impShort) + nimState.addPragma(root, impCPragma, "cdecl") if nimState.gState.dynlib.nBl: - nimState.addPragma(root, impCPragma, { - "dynlib": nimState.getIdent(nimState.gState.dynlib) - }.toOrderedTable()) + nimState.addPragma(root, impCPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib)) nimState.pragmaSection.add impPragma nimState.pragmaSection.add impCPragma From 0d1445a8ac26bd842edf491a780fffc515fa1066 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 18 Mar 2020 23:47:00 -0500 Subject: [PATCH 009/255] ast2 getNameInfo cleanup, initial skip/override support --- nimterop/ast2.nim | 155 +++++++++++++++++++++++++----------------- nimterop/getters.nim | 35 +++++----- tests/include/tast2.h | 4 +- tests/tast2.nim | 3 +- 4 files changed, 116 insertions(+), 81 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4539f79..80e409e 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -50,27 +50,55 @@ proc getLit*(nimState: NimState, str: string): PNode = if result.isNil: result = newNode(nkNilLit) -proc newConstDef(nimState: NimState, node: TSNode, name = "", val = ""): PNode = +proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode = + let + override = nimState.getOverride(origname, kind) + def = + if kind == nskConst: + nkConstDef + elif kind == nskType: + nkTypeDef + else: + nkEmpty + skind = + if kind == nskConst: + "const " + elif kind == nskType: + "type " + else: + "" + if override.nBl: + result = newNode(def) + result.add nimState.parseString(nimState.getComments()) + result.add nimState.parseString(skind & override)[0] + else: + result = nimState.parseString(nimState.getComments()) + result.add nimState.parseString(&" # $1'{origname}' skipped" % skind) + if nimState.gState.debug: + nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" + +proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode = let # node[0] = identifier = const name - (cname, info) = nimState.getNameInfo(node.getAtom(), nskConst) + (name, origname, info) = nimState.getNameInfo(node.getAtom(), nskConst) - # TODO - check blank and override ident = - if name.len != 0: - nimState.getIdent(name, info) + if fname.nBl: + nimState.getIdent(fname, info) else: - nimState.getIdent(cname, info) + nimState.getIdent(name, info) # node[1] = preproc_arg = value - nval = - if val.len != 0: - newStrNode(nkStrLit, val) + val = + if fval.nBl: + newStrNode(nkStrLit, fval) else: nimState.getLit(nimState.getNodeVal(node[1])) - # If supported literal - if nval.kind != nkNilLit: + if name.Bl and fname.Bl: + # Name skipped or overridden since blank + result = nimState.getOverrideOrSkip(node, origname, nskConst) + elif val.kind != nkNilLit and nimState.addNewIdentifer(name): # const X* = Y # # nkConstDef( @@ -84,7 +112,7 @@ proc newConstDef(nimState: NimState, node: TSNode, name = "", val = ""): PNode = result = newNode(nkConstDef) result.add ident result.add newNode(nkEmpty) - result.add nval + result.add val proc addConst(nimState: NimState, node: TSNode) = # #define X Y @@ -110,7 +138,7 @@ proc addConst(nimState: NimState, node: TSNode) = proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, value: PNode = nil) = # Add pragma to an existing nkPragma tree let - (_, pinfo) = nimState.getNameInfo(node.getAtom(), nskUnknown) + pinfo = nimState.getLineInfo(node.getAtom()) pident = nimState.getIdent(name, pinfo, exported = false) if value.isNil: @@ -173,36 +201,41 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: Orde result.add ident result.add nimState.newPragma(node, pragmas) -proc newTypeIdent(nimState: NimState, node: TSNode, override = "", union = false): PNode = +proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): PNode = # Create nkTypeDef PNode with first ident # - # If `override`, use it instead of node.getAtom() for name + # If `fname`, use it instead of node.getAtom() for name let - (name, info) = nimState.getNameInfo(node.getAtom(), nskType) - # TODO - check blank and override + (name, origname, info) = nimState.getNameInfo(node.getAtom(), nskType) + ident = - if override.len != 0: - nimState.getIdent(override, info) + if fname.nBl: + nimState.getIdent(fname, info) else: nimState.getIdent(name, info) + prident = if union: nimState.newPragmaExpr(node, ident, "union") else: ident - # type name* = - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent(name) - # ), - # nkEmpty() - # ) - result = newNode(nkTypeDef) - result.add prident - result.add newNode(nkEmpty) + if name.Bl and fname.Bl: + # Name skipped or overridden since blank + result = nimState.getOverrideOrSkip(node, origname, nskType) + else: + # type name* = + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkEmpty() + # ) + result = newNode(nkTypeDef) + result.add prident + result.add newNode(nkEmpty) proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count @@ -247,7 +280,7 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # Create nkBracketExpr tree depending on input let - (_, info) = nimState.getNameInfo(node.getAtom(), nskType) + info = nimState.getLineInfo(node.getAtom()) ident = nimState.getIdent("array", info, exported = false) # array[size, typ] @@ -296,7 +329,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn start = getStartAtom(node) # node[start] - param type - (tname, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) + (tname, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) tident = nimState.getIdent(tname, tinfo, exported = false) if start == node.len - 1: @@ -330,7 +363,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn else: # Named param, simple type let - (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) count = node[start+1].getPtrCount() @@ -343,7 +376,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn elif not fdecl.isNil: # Named param, function pointer let - (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeProc(name, node) @@ -351,7 +384,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn elif not adecl.isNil: # Named param, array type let - (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) + (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeArray(node) @@ -421,15 +454,14 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = if not field.isNil: result.add field -proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "", union = false) = +proc addTypeObject(nimState: NimState, node: TSNode, fname = "", duplicate = "", union = false) = # Add a type of object # - # If `override` is set, use it as the name + # If `fname` is set, use it as the name # If `duplicate` is set, don't add the same name decho("addTypeObject()") let - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node, override, union = union) + typeDef = nimState.newTypeIdent(node, fname, union = union) name = $typeDef[0][1] if name != duplicate: @@ -467,10 +499,10 @@ proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = nimState.printDebug(typeDef) -proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = "") = +proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", duplicate = "") = # Add a type of a specified type # - # If `toverride` is set, use it as the type name + # If `ftname` is set, use it as the type name # If `duplicate` is set, don't add the same name decho("addTypeTyped()") let @@ -479,14 +511,18 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = # Add a type of a specific type let # node[i] = identifer = name - # TODO - check blank and override typeDef = nimState.newTypeIdent(node[i]) + name = $typeDef[0][1] # node[start] = identifier = type name - (tname0, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) + (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) # Override type name - tname = if toverride.len != 0: toverride else: tname0 + tname = + if ftname.nBl: + ftname + else: + tname0 ident = nimState.getIdent(tname, tinfo, exported = false) @@ -527,7 +563,7 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = start = getStartAtom(node) # node[start] = identifier = type name - (name, info) = nimState.getNameInfo(node[start].getAtom(), nskType) + (name, origname, info) = nimState.getNameInfo(node[start].getAtom(), nskType) ident = nimState.getIdent(name, info, exported = false) # Top-most array declarator @@ -578,7 +614,6 @@ proc addTypeArray(nimState: NimState, node: TSNode) = decho("addTypeArray()") let # node[1] = identifer = name - # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) typ = nimState.getTypeArray(node) @@ -613,7 +648,7 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # Create proc type tree let # node[0] = identifier = return type name - (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + (rname, _, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType, parent = name) # Parameter list plist = node[1].anyChildInTree("parameter_list") @@ -652,7 +687,6 @@ proc addTypeProc(nimState: NimState, node: TSNode) = decho("addTypeProc()") let # node[1] = identifier = name - # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) name = $typeDef[0][1] @@ -732,7 +766,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") - if (fdlist.isNil or (not fdlist.isNil and fdlist.len == 0)) and + if (fdlist.isNil or (not fdlist.isNil and fdlist.Bl)) and nimState.getNodeVal(node[1]) == "": # typedef struct X; # @@ -882,11 +916,11 @@ proc addType(nimState: NimState, node: TSNode, union = false) = name # Now add struct as object with specified name - nimState.addTypeObject(node[0], override = name, union = union) + nimState.addTypeObject(node[0], fname = name, union = union) - if name.len != 0: + if name.nBl: # Add any additional names except duplicate - nimState.addTypeTyped(node, toverride = name, duplicate = name) + nimState.addTypeTyped(node, ftname = name, duplicate = name) proc addEnum(nimState: NimState, node: TSNode) = decho("addEnum()") @@ -900,20 +934,19 @@ proc addProc(nimState: NimState, node: TSNode) = let start = getStartAtom(node) - # node[start] = identifier = return type name - (rname, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) - - # Parameter list - plist = node[start+1].anyChildInTree("parameter_list") - # node[start+1] = identifier = name - # TODO - check blank and override ident = nimState.newTypeIdent(node[start+1]) name = $ident[0][1] # node[start+1] could have nested pointers tcount = node[start+1].getPtrCount() + # node[start] = identifier = return type name + (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + + # Parameter list + plist = node[start+1].anyChildInTree("parameter_list") + procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} @@ -1036,7 +1069,7 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = if nimState.includeHeader(): nimState.constSection.add nimState.newConstDef( - root, name = nimState.currentHeader, val = fullpath) + root, fname = nimState.currentHeader, fval = fullpath) nimState.addPragma(root, impPragma, "header", newStrNode(nkStrLit, nimState.currentHeader)) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 07ed194..736e0db 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -437,30 +437,31 @@ proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col) proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode = - # Get ident PNode for name + info - let - exp = getIdent(nimState.identCache, "*") - ident = getIdent(nimState.identCache, name) + if name.nBl: + # Get ident PNode for name + info + let + exp = getIdent(nimState.identCache, "*") + ident = getIdent(nimState.identCache, name) - if exported: - result = newNode(nkPostfix) - result.add newIdentNode(exp, info) - result.add newIdentNode(ident, info) - else: - result = newIdentNode(ident, info) + if exported: + result = newNode(nkPostfix) + result.add newIdentNode(exp, info) + result.add newIdentNode(ident, info) + else: + result = newIdentNode(ident, info) proc getIdent*(nimState: NimState, name: string): PNode = nimState.getIdent(name, nimState.getDefaultLineInfo(), exported = false) proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): - tuple[name: string, info: TLineInfo] = + tuple[name, origname: string, info: TLineInfo] = # Shortcut to get identifier name and info (node value and line:col) - let - name = nimState.getNodeVal(node) - result.name = nimState.getIdentifier(name, kind, parent) - if kind == nskType: - result.name = result.name.getType() - result.info = nimState.getLineInfo(node) + result.origname = nimState.getNodeVal(node) + result.name = nimState.getIdentifier(result.origname, kind, parent) + if result.name.nBl: + if kind == nskType: + result.name = result.name.getType() + result.info = nimState.getLineInfo(node) proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 200b4c8..7c595e4 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -15,7 +15,7 @@ typedef int *A6; typedef A0 **A7; typedef void *A8; -typedef char *A9[3]; +typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; @@ -41,4 +41,4 @@ typedef union U2 { int **f1; int abc[123+132]; } U2; // Anonymous //typedef struct { char a1; }; -struct A2 test_proc1(struct A0 a); \ No newline at end of file +//struct A2 test_proc1(struct A0 a); \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index d8d18a3..9e32f6e 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -40,7 +40,8 @@ assert A6 is ptr cint assert A7 is ptr ptr A0 assert A8 is pointer -assert A9 is array[3, cstring] +assert A9p is array[3, cstring] +#assert A9 is array[4, cchar] assert A10 is array[3, array[6, cstring]] assert A11 is ptr array[3, cstring] From f01c7ea60dab54c14a9d44b359dcb025812470bc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 20 Mar 2020 01:31:54 -0500 Subject: [PATCH 010/255] ast2 improve override/skip, duplicates --- nimterop/ast2.nim | 442 +++++++++++++++++++++++------------------- nimterop/globals.nim | 3 +- tests/include/tast2.h | 53 +++++ tests/tast2.nim | 12 +- 4 files changed, 304 insertions(+), 206 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 80e409e..95ccdd6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -8,8 +8,6 @@ import "."/treesitter/api import "."/[globals, getters] -# Move to getters after ast2 becomes default - proc getPtrType*(str: string): string = result = case str: of "cchar": @@ -20,6 +18,7 @@ proc getPtrType*(str: string): string = str proc parseString(nimState: NimState, str: string): PNode = + # Parse a string into Nim AST result = parseString(str, nimState.identCache, nimState.config) proc getLit*(nimState: NimState, str: string): PNode = @@ -51,70 +50,86 @@ proc getLit*(nimState: NimState, str: string): PNode = result = newNode(nkNilLit) proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode = + # Check if symbol `origname` of `kind` and `origname` has any cOverride defined + # and use that if present + # + # If not, symbol needs to be skipped - only get here if `name` is blank let + # Get cleaned name for symbol, set parent so that cOverride is ignored + name = nimState.getIdentifier(origname, kind, parent = "override") + override = nimState.getOverride(origname, kind) - def = - if kind == nskConst: - nkConstDef - elif kind == nskType: - nkTypeDef - else: - nkEmpty skind = if kind == nskConst: "const " elif kind == nskType: "type " + elif kind == nskProc: + "proc " else: "" + if override.nBl: - result = newNode(def) - result.add nimState.parseString(nimState.getComments()) - result.add nimState.parseString(skind & override)[0] + result = nimState.parseString(skind & override.replace(origname, name))[0][0] else: - result = nimState.parseString(nimState.getComments()) - result.add nimState.parseString(&" # $1'{origname}' skipped" % skind) + necho &"\n# $1'{origname}' skipped" % skind if nimState.gState.debug: nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode = + # Create an nkConstDef PNode + # + # If `fname` or `fval` are set, use them as name and val let # node[0] = identifier = const name - (name, origname, info) = nimState.getNameInfo(node.getAtom(), nskConst) + (cname, origname, info) = nimState.getNameInfo(node.getAtom(), nskConst) - ident = + name = if fname.nBl: - nimState.getIdent(fname, info) + fname else: - nimState.getIdent(name, info) + cname + ident = nimState.getIdent(name, info) # node[1] = preproc_arg = value val = + if fval.nBl: + fval + else: + nimState.getNodeVal(node[1]) + valident = if fval.nBl: newStrNode(nkStrLit, fval) else: - nimState.getLit(nimState.getNodeVal(node[1])) + nimState.getLit(val) - if name.Bl and fname.Bl: + if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif val.kind != nkNilLit and nimState.addNewIdentifer(name): - # const X* = Y - # - # nkConstDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkXLit(Y) - # ) - result = newNode(nkConstDef) - result.add ident - result.add newNode(nkEmpty) - result.add val + elif valident.kind != nkNilLit: + if nimState.addNewIdentifer(name): + # const X* = Y + # + # nkConstDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkXLit(Y) + # ) + result = newNode(nkConstDef) + result.add ident + result.add newNode(nkEmpty) + result.add valident + else: + necho &"# const '{origname}' is duplicate, skipped" + else: + necho &"# const '{origname}' has invalid value '{val}'" proc addConst(nimState: NimState, node: TSNode) = + # Add a const to the AST + # # #define X Y # # (preproc_def @@ -195,7 +210,7 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, name: string, result.add ident result.add nimState.newPragma(node, name, value) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode = +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode {.used.} = # Create nkPragmaExpr tree for multiple name:value result = newNode(nkPragmaExpr) result.add ident @@ -206,13 +221,14 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): # # If `fname`, use it instead of node.getAtom() for name let - (name, origname, info) = nimState.getNameInfo(node.getAtom(), nskType) + (tname, origname, info) = nimState.getNameInfo(node.getAtom(), nskType) - ident = + name = if fname.nBl: - nimState.getIdent(fname, info) + fname else: - nimState.getIdent(name, info) + tname + ident = nimState.getIdent(name, info) prident = if union: @@ -220,10 +236,10 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): else: ident - if name.Bl and fname.Bl: + if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskType) - else: + elif nimState.addNewIdentifer(name): # type name* = # # nkTypeDef( @@ -236,6 +252,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): result = newNode(nkTypeDef) result.add prident result.add newNode(nkEmpty) + else: + necho &"# type '{origname}' is duplicate, skipped" proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count @@ -295,7 +313,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add size result.add typ -proc getTypeArray(nimState: NimState, node: TSNode): PNode +proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = @@ -387,7 +405,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeArray(node) + result.add nimState.getTypeArray(node, name) result.add newNode(nkEmpty) else: result = nil @@ -454,17 +472,20 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = if not field.isNil: result.add field -proc addTypeObject(nimState: NimState, node: TSNode, fname = "", duplicate = "", union = false) = +proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", union = false) = # Add a type of object # - # If `fname` is set, use it as the name - # If `duplicate` is set, don't add the same name + # If `typeDef` is set, use it instead of creating new PNode + # If `fname` is set, use it as the name when creating new PNode decho("addTypeObject()") let - typeDef = nimState.newTypeIdent(node, fname, union = union) - name = $typeDef[0][1] + typeDef = + if typeDef.isNil: + nimState.newTypeIdent(node, fname, union = union) + else: + typeDef - if name != duplicate: + if not typeDef.isNil: # type X* = object # # nkTypeDef( @@ -480,6 +501,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, fname = "", duplicate = "", # ) # ) let + name = $typeDef[0][1] obj = newNode(nkObjectTy) obj.add newNode(nkEmpty) obj.add newNode(nkEmpty) @@ -499,72 +521,75 @@ proc addTypeObject(nimState: NimState, node: TSNode, fname = "", duplicate = "", nimState.printDebug(typeDef) -proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", duplicate = "") = +proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specified type # # If `ftname` is set, use it as the type name - # If `duplicate` is set, don't add the same name + # If `offset` is set, skip `offset` names, since created already decho("addTypeTyped()") let start = getStartAtom(node) - for i in start+1 ..< node.len: + for i in start+1+offset ..< node.len: # Add a type of a specific type let # node[i] = identifer = name typeDef = nimState.newTypeIdent(node[i]) - name = $typeDef[0][1] + + if not typeDef.isNil: + let + name = $typeDef[0][1] - # node[start] = identifier = type name - (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + # node[start] = identifier = type name + (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) - # Override type name - tname = - if ftname.nBl: - ftname + # Override type name + tname = + if ftname.nBl: + ftname + else: + tname0 + + ident = nimState.getIdent(tname, tinfo, exported = false) + + # node[i] could have nested pointers + count = node[i].getPtrCount() + + # Skip typedef X X; + if $typeDef[0][1] != tname: + if count > 0: + # If pointers + typeDef.add nimState.newPtrTree(count, ident) else: - tname0 + typeDef.add ident - ident = nimState.getIdent(tname, tinfo, exported = false) + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) - # node[i] could have nested pointers - count = node[i].getPtrCount() + # nkTypeSection.add + nimState.typeSection.add typeDef - # Skip typedef X X; - if $typeDef[0][1] != tname: - if count > 0: - # If pointers - typeDef.add nimState.newPtrTree(count, ident) + nimState.printDebug(typeDef) else: - typeDef.add ident + nimState.addTypeObject(node, typeDef = typeDef) - # type X* = [ptr ..] Y - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkIdent("Y") - # ) - # ) - - # nkTypeSection.add - nimState.typeSection.add typeDef - - nimState.printDebug(typeDef) - else: - nimState.addTypeObject(node, duplicate = duplicate) - -proc getTypeArray(nimState: NimState, node: TSNode): PNode = +proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode = # Create array type tree let start = getStartAtom(node) # node[start] = identifier = type name - (name, origname, info) = nimState.getNameInfo(node[start].getAtom(), nskType) - ident = nimState.getIdent(name, info, exported = false) + (tname, _, info) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + ident = nimState.getIdent(tname, info, exported = false) # Top-most array declarator adecl = node[start+1].firstChildInTree("array_declarator") @@ -616,33 +641,36 @@ proc addTypeArray(nimState: NimState, node: TSNode) = # node[1] = identifer = name typeDef = nimState.newTypeIdent(node[1]) - typ = nimState.getTypeArray(node) + if not typeDef.isNil: + let + name = $typeDef[0][1] + typ = nimState.getTypeArray(node, name) - typeDef.add typ + typeDef.add typ - # type X* = [ptr] array[x, [ptr] Y] - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkBracketExpr( - # nkIdent("array") - # nkXLit(x), - # nkPtrTy( # optional, nested - # nkIdent("Y") - # ) - # ) - # ) - # ) + # type X* = [ptr] array[x, [ptr] Y] + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkBracketExpr( + # nkIdent("array") + # nkXLit(x), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # ) + # ) - # nkTypeSection.add - nimState.typeSection.add typeDef + # nkTypeSection.add + nimState.typeSection.add typeDef - nimState.printDebug(typeDef) + nimState.printDebug(typeDef) proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # Create proc type tree @@ -688,44 +716,47 @@ proc addTypeProc(nimState: NimState, node: TSNode) = let # node[1] = identifier = name typeDef = nimState.newTypeIdent(node[1]) - name = $typeDef[0][1] - procTy = nimState.getTypeProc(name, node) + if not typeDef.isNil: + let + name = $typeDef[0][1] - typeDef.add procTy + procTy = nimState.getTypeProc(name, node) - # type X* = proc(a1: Y, a2: Z): P - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkProcTy( - # nkFormalParams( - # nkPtrTy( # optional, nested - # nkIdent(retType) - # ), - # nkIdentDefs( - # nkIdent(param), - # nkPtrTy( - # nkIdent(ptype) - # ), - # nkEmpty() - # ), - # ... - # ), - # nkEmpty() - # ) - # ) - # ) + typeDef.add procTy - # nkTypeSection.add - nimState.typeSection.add typeDef + # type X* = proc(a1: Y, a2: Z): P + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkProcTy( + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkEmpty() + # ) + # ) + # ) - nimState.printDebug(typeDef) + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode, union = false) = decho("addType()") @@ -897,7 +928,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = if node.len > 1 and nimState.getNodeVal(node[1]) != "": # Add any additional names - nimState.addTypeTyped(node, duplicate = nimState.getNodeVal(node[0].getAtom())) + nimState.addTypeTyped(node) else: # Same as above except unnamed struct # @@ -919,8 +950,8 @@ proc addType(nimState: NimState, node: TSNode, union = false) = nimState.addTypeObject(node[0], fname = name, union = union) if name.nBl: - # Add any additional names except duplicate - nimState.addTypeTyped(node, ftname = name, duplicate = name) + # Add any additional names + nimState.addTypeTyped(node, ftname = name, offset = 1) proc addEnum(nimState: NimState, node: TSNode) = decho("addEnum()") @@ -935,67 +966,72 @@ proc addProc(nimState: NimState, node: TSNode) = start = getStartAtom(node) # node[start+1] = identifier = name - ident = nimState.newTypeIdent(node[start+1]) - name = $ident[0][1] + tident = nimState.newTypeIdent(node[start+1]) + + if not tident.isNil: + let + # Only need the ident tree, not nkTypeDef parent + ident = tident[0] + name = $tident[0][1] - # node[start+1] could have nested pointers - tcount = node[start+1].getPtrCount() + # node[start+1] could have nested pointers + tcount = node[start+1].getPtrCount() - # node[start] = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + # node[start] = identifier = return type name + (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) - # Parameter list - plist = node[start+1].anyChildInTree("parameter_list") + # Parameter list + plist = node[start+1].anyChildInTree("parameter_list") - procDef = newNode(nkProcDef) + procDef = newNode(nkProcDef) - # proc X(a1: Y, a2: Z): P {.pragma.} - # - # nkProcDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkEmpty(), - # nkFormalParams( - # nkPtrTy( # optional, nested - # nkIdent(retType) - # ), - # nkIdentDefs( - # nkIdent(param), - # nkPtrTy( - # nkIdent(ptype) - # ), - # nkEmpty() - # ), - # ... - # ), - # nkPragma(...), - # nkEmpty(), - # nkEmpty() - # ) + # proc X(a1: Y, a2: Z): P {.pragma.} + # + # nkProcDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkEmpty(), + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkPragma(...), + # nkEmpty(), + # nkEmpty() + # ) - procDef.add ident - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) + procDef.add ident + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) - # Return type - var - retType = nimState.getIdent(rname, rinfo, exported = false) - if tcount > 0: - retType = nimState.newPtrTree(tcount, retType) + # Return type + var + retType = nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = nimState.newPtrTree(tcount, retType) - # Proc with return type and params - procDef.add nimState.newFormalParams(name, plist, retType) - procDef.add newNode(nkEmpty) # Pragmas - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) + # Proc with return type and params + procDef.add nimState.newFormalParams(name, plist, retType) + procDef.add newNode(nkEmpty) # Pragmas + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) - # nkProcSection.add - nimState.procSection.add procDef + # nkProcSection.add + nimState.procSection.add procDef - nimState.printDebug(procDef) + nimState.printDebug(procDef) proc processNode(nimState: NimState, node: TSNode): bool = result = true diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 2fb543b..12de6a7 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -120,7 +120,8 @@ when not declared(CIMPORT): gState.outputHandle.writeLine(args) template necho*(args: string) {.dirty.} = - let gState = nimState.gState + when not declared(gState): + let gState = nimState.gState gecho args template decho*(str: untyped): untyped = diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 7c595e4..5cabfac 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -1,3 +1,55 @@ +#define A 1 +#define B 1.0 +#define C 0x10 +#define D "hello" +#define E 'c' + +struct A0; +struct A1 {}; +typedef struct A2; +typedef struct A3 {}; +typedef struct A4 A4, *A4p; +typedef const int A5; +typedef int *A6; +typedef A0 **A7; +typedef void *A8; + +typedef char *A9p[3]; //, A9[4]; +typedef char *A10[3][6]; +typedef char *(*A11)[3]; +typedef struct A1 *A111[12]; + +typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int A13(int, int); + +struct A14 { volatile char a1; }; +struct A15 { char *a1; const int *a2[1]; }; + +typedef struct A16 { char f1; }; +typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p; +typedef struct { char *a1; int *a2[1]; } A19, *A19p; + +typedef struct A20 { char a1; } A20, A21, *A21p; + +//Expression +typedef struct A22 { int **f1; int *f2[123+132]; } A22; + +//Unions +union U1 {int f1; float f2; }; +typedef union U2 { int **f1; int abc[123+132]; } U2; + +// Anonymous +//typedef struct { char a1; }; + +//struct A2 test_proc1(struct A0 a); + + + + + + + +// DUPLICATES #define A 1 #define B 1.0 @@ -18,6 +70,7 @@ typedef void *A8; typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; +typedef struct A1 *A111[12]; typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); typedef int A13(int, int); diff --git a/tests/tast2.nim b/tests/tast2.nim index 9e32f6e..40c7886 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -5,6 +5,13 @@ import nimterop/[cimport] static: cDebug() +cOverride: + const + A* = 2 + + type + A1* = A0 + cImport("include/tast2.h", flags="-d -f:ast2") proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = @@ -18,7 +25,7 @@ proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, s "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value) assert count == fields.len, "Failed for " & $t -assert A == 1 +assert A == 2 assert B == 1.0 assert C == 0x10 assert D == "hello" @@ -26,7 +33,7 @@ assert E == 'c' assert A0 is object testFields(A0) -assert A1 is object +assert A1 is A0 testFields(A1) assert A2 is object testFields(A2) @@ -44,6 +51,7 @@ assert A9p is array[3, cstring] #assert A9 is array[4, cchar] assert A10 is array[3, array[6, cstring]] assert A11 is ptr array[3, cstring] +assert A111 is array[12, ptr A1] assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint assert A13 is proc(a1: cint, a2: cint): cint From 4cfeea67a1152534e620db9832785f3ee5f9b259 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 20 Mar 2020 01:41:36 -0500 Subject: [PATCH 011/255] Fix for 0.20.2 --- nimterop/getters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 736e0db..e1aa53e 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -427,7 +427,7 @@ proc printDebug*(nimState: NimState, pnode: PNode) = # Compiler shortcuts proc getDefaultLineInfo*(nimState: NimState): TLineInfo = - result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, -1, -1) + result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, 0, 0) proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = # Get Nim equivalent line:col info from node From be1e85934d39b7d0d8ef481593c0c4cbae333310 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 21 Mar 2020 12:51:12 -0500 Subject: [PATCH 012/255] ast2 forward declaration support --- nimterop/ast2.nim | 21 +++++++++++++++++++++ nimterop/globals.nim | 4 ++++ tests/include/tast2.h | 10 ++++++++++ tests/tast2.nim | 4 ++-- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 95ccdd6..c883537 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -252,6 +252,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): result = newNode(nkTypeDef) result.add prident result.add newNode(nkEmpty) + + nimState.identifierNodes[name] = result else: necho &"# type '{origname}' is duplicate, skipped" @@ -520,6 +522,24 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname nimState.typeSection.add typeDef nimState.printDebug(typeDef) + else: + # Forward declaration case + let + fdlist = node.anyChildInTree("field_declaration_list") + if not fdlist.isNil and fdlist.len > 0: + # Current node has fields + let + name = nimState.getNodeVal(node.getAtom()) + + if nimState.identifierNodes.hasKey(name): + let + def = nimState.identifierNodes[name] + # Duplicate nkTypeDef for `name` with empty fields + if def.kind == nkTypeDef and def.len == 3 and + def[2].kind == nkObjectTy and def[2].len == 3 and + def[2][2].kind == nkEmpty: + # Add fields to existing object + def[2][2] = nimState.newRecListTree(name, fdlist) proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specified type @@ -1135,6 +1155,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = fp = fullpath.replace("\\", "/") nimState.identifiers = newTable[string, string]() + nimState.identifierNodes = newTable[string, PNode]() nimState.gState = gState nimState.currentHeader = getCurrentHeader(fullpath) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 12de6a7..e7ea4c0 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -69,6 +69,7 @@ type outputHandle*: File NimState {.used.} = ref object + # All symbols that have been declared so far indexed by nimName identifiers*: TableRef[string, string] # Legacy ast fields, remove when ast2 becomes default @@ -83,6 +84,9 @@ type config*: ConfigRef graph*: ModuleGraph + # Craeted symbols to generated AST - forward declaration tracking + identifierNodes*: TableRef[string, PNode] + gState*: State currentHeader*, impShort*, sourceFile*: string diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 5cabfac..e9339ae 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -14,6 +14,11 @@ typedef int *A6; typedef A0 **A7; typedef void *A8; +// Forward declaration +struct A0 { + int f1; +}; + typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; @@ -67,6 +72,11 @@ typedef int *A6; typedef A0 **A7; typedef void *A8; +// Forward declaration +struct A0 { + int f1; +}; + typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; diff --git a/tests/tast2.nim b/tests/tast2.nim index 40c7886..6465acc 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -32,9 +32,9 @@ assert D == "hello" assert E == 'c' assert A0 is object -testFields(A0) +testFields(A0, {"f1": "cint"}.toTable()) assert A1 is A0 -testFields(A1) +testFields(A1, {"f1": "cint"}.toTable()) assert A2 is object testFields(A2) assert A3 is object From 6d1c428b6ebea47c7956c8580c26a810a9250db6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 21 Mar 2020 21:29:15 -0500 Subject: [PATCH 013/255] Remove lzma test --- tests/getheader.nims | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/getheader.nims b/tests/getheader.nims index 5a08429..7a0fc80 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -35,10 +35,6 @@ when defined(posix): testCall(cmd & " -d:zlibStd" & zrcmd, zexp, 0) testCall(cmd & " -d:zlibStd -d:zlibStatic" & zrcmd, zexp, 0) - # git - testCall(cmd & " -d:lzmaGit" & lrcmd, lexp, 0) - testCall(cmd & " -d:lzmaGit -d:lzmaStatic" & lrcmd, lexp, 0, delete = false) - # git tag testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) From cadf16293d40bcd265d1bf8f366d351ba354a6f9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 23 Mar 2020 14:02:40 -0500 Subject: [PATCH 014/255] ast2 enum support --- nimterop/ast2.nim | 84 ++++++++++++++++++++++++++++++++++--- nimterop/getters.nim | 5 ++- nimterop/globals.nim | 3 ++ tests/include/tast2.h | 96 +++++++++++++++++++++++++++++++++++++++++++ tests/tast2.nim | 21 +++++++++- 5 files changed, 200 insertions(+), 9 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index c883537..23cddcb 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -98,10 +98,7 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode else: nimState.getNodeVal(node[1]) valident = - if fval.nBl: - newStrNode(nkStrLit, fval) - else: - nimState.getLit(val) + nimState.getLit(val) if name.Bl: # Name skipped or overridden since blank @@ -147,6 +144,7 @@ proc addConst(nimState: NimState, node: TSNode) = if not constDef.isNil: # nkConstSection.add nimState.constSection.add constDef + nimState.constIdentifiers.incl $constDef[0][1] nimState.printDebug(constDef) @@ -977,6 +975,80 @@ proc addEnum(nimState: NimState, node: TSNode) = decho("addEnum()") nimState.printDebug(node) + let + enumlist = node.anyChildInTree("enumerator_list") + if not enumlist.isNil: + var + name, origname = "" + offset = 0 + prev = "" + + if node.getAtom().getName() == "type_identifier": + # [typedef] enum X {} Y; + # Use X as name + origname = nimState.getNodeVal(node.getAtom()) + elif node.getName() == "type_definition" and node.len > 1: + # typedef enum {} Y; + # Use Y as name + origname = nimState.getNodeVal(node[1].getAtom()) + offset = 1 + + if origname.nBl: + name = nimState.getIdentifier(origname, nskType) + else: + # enum {}; + # Nameless so create a name + name = nimState.getUniqueIdentifier("Enum") + + if name.Bl: + # Name skipped or overridden since blank + let + eoverride = nimState.getOverrideOrSkip(node, origname, nskType) + if not eoverride.isNil: + nimState.typeSection.add eoverride + elif nimState.addNewIdentifer(name): + # Add enum definition and helpers + nimState.enumSection.add nimState.parseString(&"defineEnum({name})") + + # Create const for fields + var + fnames: HashSet[string] + for i in 0 .. enumlist.len - 1: + let + en = enumlist[i] + if en.getName() == "comment": + continue + let + fname = nimState.getIdentifier(nimState.getNodeVal(en.getAtom()), nskEnumField) + if fname.nBl: + var + fval = "" + if prev.Bl: + # Starting default value + fval = &"(0).{name}" + else: + # One greater than previous + fval = &"({prev} + 1).{name}" + + if en.len > 1 and en[1].getName() in gEnumVals: + # Explicit value + fval = "(" & nimState.getNimExpression(nimState.getNodeVal(en[1]), name) & ")." & name + + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + + fnames.incl fname + + prev = fname + + # Add fields to list of consts after processing enum so that we don't cast + # enum field to itself + nimState.constIdentifiers.incl fnames + + # Add other names + if node.getName() == "type_definition" and node.len > 1: + nimState.addTypeTyped(node, ftname = name, offset = offset) + proc addProc(nimState: NimState, node: TSNode) = # Add a proc decho("addProc()") @@ -1060,9 +1132,9 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "preproc_def": nimState.addConst(node) of "type_definition": - if not node.firstChildInTree("enum_specifier").isNil(): + if node.len > 0 and node[0].getName() == "enum_specifier": nimState.addEnum(node) - elif not node.firstChildInTree("union_specifier").isNil(): + elif node.len > 0 and node[0].getName() == "union_specifier": nimState.addType(node, union = true) else: nimState.addType(node) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index e1aa53e..091174b 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -596,7 +596,8 @@ proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = if ast.children.len == 1 and ast.children[0].name == ".": return ast.children[0] -proc getNimExpression*(nimState: NimState, expr: string): string = +proc getNimExpression*(nimState: NimState, expr: string, name = ""): string = + # Convert C/C++ expression into Nim - cast identifiers to `name` if specified var clean = expr.multiReplace([("\n", " "), ("\r", "")]) ident = "" @@ -641,6 +642,8 @@ proc getNimExpression*(nimState: NimState, expr: string): string = # Process identifier if ident.nBl: ident = nimState.getIdentifier(ident, nskConst) + if name.nBl and ident in nimState.constIdentifiers: + ident = ident & "." & name result &= ident ident = "" result &= gen diff --git a/nimterop/globals.nim b/nimterop/globals.nim index e7ea4c0..68f18b0 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -72,6 +72,9 @@ type # All symbols that have been declared so far indexed by nimName identifiers*: TableRef[string, string] + # All const names for enum casting + constIdentifiers*: HashSet[string] + # Legacy ast fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e9339ae..b867020 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -43,6 +43,54 @@ typedef struct A22 { int **f1; int *f2[123+132]; } A22; union U1 {int f1; float f2; }; typedef union U2 { int **f1; int abc[123+132]; } U2; +// Enums + +// Issue #159 +#define NK_FLAG(x) (1 << (x)) +enum nk_panel_type { + NK_PANEL_NONE = 0, + NK_PANEL_WINDOW = NK_FLAG(0), + NK_PANEL_GROUP = NK_FLAG(1), + NK_PANEL_POPUP = NK_FLAG(2), + NK_PANEL_CONTEXTUAL = NK_FLAG(4), + NK_PANEL_COMBO = NK_FLAG(5), + NK_PANEL_MENU = NK_FLAG(6), + NK_PANEL_TOOLTIP = NK_FLAG(7) +}; +enum nk_panel_set { + NK_PANEL_SET_NONBLOCK = NK_PANEL_CONTEXTUAL|NK_PANEL_COMBO|NK_PANEL_MENU|NK_PANEL_TOOLTIP, + NK_PANEL_SET_POPUP = NK_PANEL_SET_NONBLOCK|NK_PANEL_POPUP, + NK_PANEL_SET_SUB = NK_PANEL_SET_POPUP|NK_PANEL_GROUP +}; + +// Issue #171 +typedef enum VSColorFamily { + /* all planar formats */ + cmGray = 1000000, + cmRGB = 2000000, + cmYUV = 3000000, + cmYCoCg = 4000000, + /* special for compatibility */ + cmCompat = 9000000 +} VSColorFamily; + +typedef enum VSPresetFormat { + pfNone = 0, + + pfGray8 = cmGray + 10, + pfGray16, + + pfYUV420P8 = cmYUV + 10, + pfYUV422P8, + + pfRGB24 = cmRGB + 10, + pfRGB27, + /* test */ + + pfCompatBGR32 = cmCompat + 10, + pfCompatYUY2 +} VSPresetFormat; + // Anonymous //typedef struct { char a1; }; @@ -101,6 +149,54 @@ typedef struct A22 { int **f1; int *f2[123+132]; } A22; union U1 {int f1; float f2; }; typedef union U2 { int **f1; int abc[123+132]; } U2; +// Enums + +// Issue #159 +#define NK_FLAG(x) (1 << (x)) +enum nk_panel_type { + NK_PANEL_NONE = 0, + NK_PANEL_WINDOW = NK_FLAG(0), + NK_PANEL_GROUP = NK_FLAG(1), + NK_PANEL_POPUP = NK_FLAG(2), + NK_PANEL_CONTEXTUAL = NK_FLAG(4), + NK_PANEL_COMBO = NK_FLAG(5), + NK_PANEL_MENU = NK_FLAG(6), + NK_PANEL_TOOLTIP = NK_FLAG(7) +}; +enum nk_panel_set { + NK_PANEL_SET_NONBLOCK = NK_PANEL_CONTEXTUAL|NK_PANEL_COMBO|NK_PANEL_MENU|NK_PANEL_TOOLTIP, + NK_PANEL_SET_POPUP = NK_PANEL_SET_NONBLOCK|NK_PANEL_POPUP, + NK_PANEL_SET_SUB = NK_PANEL_SET_POPUP|NK_PANEL_GROUP +}; + +// Issue #171 +typedef enum VSColorFamily { + /* all planar formats */ + cmGray = 1000000, + cmRGB = 2000000, + cmYUV = 3000000, + cmYCoCg = 4000000, + /* special for compatibility */ + cmCompat = 9000000 +} VSColorFamily; + +typedef enum VSPresetFormat { + pfNone = 0, + + pfGray8 = cmGray + 10, + pfGray16, + + pfYUV420P8 = cmYUV + 10, + pfYUV422P8, + + pfRGB24 = cmRGB + 10, + pfRGB27, + /* test */ + + pfCompatBGR32 = cmCompat + 10, + pfCompatYUY2 +} VSPresetFormat; + // Anonymous //typedef struct { char a1; }; diff --git a/tests/tast2.nim b/tests/tast2.nim index 6465acc..914d40f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -12,7 +12,7 @@ cOverride: type A1* = A0 -cImport("include/tast2.h", flags="-d -f:ast2") +cImport("include/tast2.h", flags="-d -f:ast2 -ENK_") proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = var @@ -86,4 +86,21 @@ assert U1 is object assert sizeof(U1) == sizeof(cfloat) assert U2 is object -assert sizeof(U2) == 256 * sizeof(cint) \ No newline at end of file +assert sizeof(U2) == 256 * sizeof(cint) + +assert PANEL_WINDOW == 1 +assert PANEL_GROUP == 2 +assert PANEL_POPUP == 4 +assert PANEL_CONTEXTUAL == 16 +assert PANEL_COMBO == 32 +assert PANEL_MENU == 64 +assert PANEL_TOOLTIP == 128 +assert PANEL_SET_NONBLOCK == 240 +assert PANEL_SET_POPUP == 244 +assert PANEL_SET_SUB == 246 + +assert cmGray == 1000000 +assert pfGray16 == 1000011 +assert pfYUV422P8 == pfYUV420P8 + 1 +assert pfRGB27 == cmRGB.VSPresetFormat + 11 +assert pfCompatYUY2 == pfCompatBGR32 + 1 \ No newline at end of file From c113ecec986e4790a98c5c73f61864fe18c49bfe Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 23 Mar 2020 16:06:28 -0500 Subject: [PATCH 015/255] ast2 bug fixes - getIdentName, ptr object, header const --- nimterop/ast2.nim | 40 +++++++++++++++++++--------------------- nimterop/getters.nim | 8 ++++++++ tests/tnimterop_c.nim | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 23cddcb..6c2589c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -81,14 +81,15 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode # # If `fname` or `fval` are set, use them as name and val let - # node[0] = identifier = const name - (cname, origname, info) = nimState.getNameInfo(node.getAtom(), nskConst) - - name = + origname = if fname.nBl: fname else: - cname + # node[0] = identifier = const name + nimState.getNodeVal(node.getAtom()) + + name = nimState.getIdentifier(origname, nskConst) + info = nimState.getLineInfo(node) ident = nimState.getIdent(name, info) # node[1] = preproc_arg = value @@ -144,7 +145,7 @@ proc addConst(nimState: NimState, node: TSNode) = if not constDef.isNil: # nkConstSection.add nimState.constSection.add constDef - nimState.constIdentifiers.incl $constDef[0][1] + nimState.constIdentifiers.incl constDef.getIdentName() nimState.printDebug(constDef) @@ -259,9 +260,9 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring + result = typ var count = count - chng = false if typ.kind == nkIdent: let tname = typ.ident.s @@ -269,7 +270,6 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = if tname != ptname: # If Nim type available, use that ident result = nimState.getIdent(ptname, typ.info, exported = false) - chng = true # One ptr reduced count -= 1 if count > 0: @@ -282,18 +282,16 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # typ # ) # ) - result = newNode(nkPtrTy) var - parent = result + nresult = newNode(nkPtrTy) + parent = nresult child: PNode for i in 1 ..< count: child = newNode(nkPtrTy) parent.add child parent = child - parent.add typ - elif not chng: - # Either no ptr, or none left after Nim type adjustment - result = typ + parent.add result + result = nresult proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # Create nkBracketExpr tree depending on input @@ -501,7 +499,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname # ) # ) let - name = $typeDef[0][1] + name = typeDef.getIdentName() obj = newNode(nkObjectTy) obj.add newNode(nkEmpty) obj.add newNode(nkEmpty) @@ -555,7 +553,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = if not typeDef.isNil: let - name = $typeDef[0][1] + name = typeDef.getIdentName() # node[start] = identifier = type name (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) @@ -573,7 +571,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = count = node[i].getPtrCount() # Skip typedef X X; - if $typeDef[0][1] != tname: + if name != tname: if count > 0: # If pointers typeDef.add nimState.newPtrTree(count, ident) @@ -661,7 +659,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = if not typeDef.isNil: let - name = $typeDef[0][1] + name = typeDef.getIdentName() typ = nimState.getTypeArray(node, name) typeDef.add typ @@ -737,7 +735,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = if not typeDef.isNil: let - name = $typeDef[0][1] + name = typeDef.getIdentName() procTy = nimState.getTypeProc(name, node) @@ -1064,7 +1062,7 @@ proc addProc(nimState: NimState, node: TSNode) = let # Only need the ident tree, not nkTypeDef parent ident = tident[0] - name = $tident[0][1] + name = tident.getIdentName() # node[start+1] could have nested pointers tcount = node[start+1].getPtrCount() @@ -1197,7 +1195,7 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = if nimState.includeHeader(): nimState.constSection.add nimState.newConstDef( - root, fname = nimState.currentHeader, fval = fullpath) + root, fname = nimState.currentHeader, fval = '"' & fullpath & '"') nimState.addPragma(root, impPragma, "header", newStrNode(nkStrLit, nimState.currentHeader)) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 091174b..251955d 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -453,6 +453,14 @@ proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = tru proc getIdent*(nimState: NimState, name: string): PNode = nimState.getIdent(name, nimState.getDefaultLineInfo(), exported = false) +proc getIdentName*(node: PNode): string = + if not node.isNil: + for i in 0 ..< node.len: + if node[i].kind == nkIdent and $node[i] != "*": + result = $node[i] + if result.Bl and node.len > 0: + result = node[0].getIdentName() + proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): tuple[name, origname: string, info: TLineInfo] = # Shortcut to get identifier name and info (node value and line:col) diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index c577c5d..732efbf 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -41,7 +41,7 @@ cOverride: proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.} proc weirdfunc2(mango: ptr ptr cchar): int {.importc.} -cImport cSearchPath "test.h" +cImport(cSearchPath("test.h")) check TEST_INT == 512 check TEST_FLOAT == 5.12 From c5e978c2efe8557862944a7471eb9a3356b0f3f4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 23 Mar 2020 22:07:39 -0500 Subject: [PATCH 016/255] ast2 override final, void return and param fixes, multi proc support --- nimterop/ast2.nim | 192 ++++++++++++++++++++++++++---------------- nimterop/getters.nim | 7 ++ tests/include/tast2.h | 2 +- tests/tast2.nim | 2 +- 4 files changed, 128 insertions(+), 75 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6c2589c..d3d32c3 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,4 +1,4 @@ -import macros, os, sets, strformat, strutils, tables, times +import macros, os, sequtils, sets, strformat, strutils, tables, times import regex @@ -59,23 +59,51 @@ proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: name = nimState.getIdentifier(origname, kind, parent = "override") override = nimState.getOverride(origname, kind) - skind = - if kind == nskConst: - "const " - elif kind == nskType: - "type " - elif kind == nskProc: - "proc " - else: - "" + var + skind = getKeyword(kind) & " " if override.nBl: + if kind == nskProc: + skind = "" result = nimState.parseString(skind & override.replace(origname, name))[0][0] else: necho &"\n# $1'{origname}' skipped" % skind if nimState.gState.debug: nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" +proc addOverrideFinal(nimState: NimState, kind: NimSymKind) = + # Add all unused cOverride symbols for `kind` to AST + var + syms = nimState.getOverrideFinal(kind) + skind = getKeyword(kind) & "\n" + if kind == nskProc: + skind = "" + + if syms.nBl: + var + nsyms = nimState.parseString(skind & syms) + if not nsyms.isNil: + let + list = + if kind == nskProc: + nsyms.sons + else: + nsyms[0].sons + case kind + of nskConst: + nimState.constSection.sons.insert(list, 0) + of nskType: + nimState.typeSection.sons.insert(list, 0) + of nskProc: + nimState.procSection.sons.insert(list, 0) + else: + discard + +proc addAllOverrideFinal(nimState: NimState) = + # Add all unused cOverride symbols to AST + for kind in [nskConst, nskType, nskProc]: + nimState.addOverrideFinal(kind) + proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode = # Create an nkConstDef PNode # @@ -352,12 +380,16 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn # Only for proc with no named param - create a param name based on offset # # int func(char, int); - let - pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported) - result.add pident - result.add tident - result.add newNode(nkEmpty) + if tname != "object": + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported) + result.add pident + result.add tident + result.add newNode(nkEmpty) + else: + # int func(void) + result = nil else: let fdecl = node[start+1].anyChildInTree("function_declarator") @@ -717,7 +749,12 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # Return type var - retType = nimState.getIdent(rname, rinfo, exported = false) + retType = + if rname == "object" and tcount == 0: + # void (*func)(..) + newNode(nkEmpty) + else: + nimState.getIdent(rname, rinfo, exported = false) if tcount > 0: retType = nimState.newPtrTree(tcount, retType) @@ -1055,73 +1092,80 @@ proc addProc(nimState: NimState, node: TSNode) = let start = getStartAtom(node) - # node[start+1] = identifier = name - tident = nimState.newTypeIdent(node[start+1]) - - if not tident.isNil: + for i in start+1 ..< node.len: let - # Only need the ident tree, not nkTypeDef parent - ident = tident[0] - name = tident.getIdentName() + # node[i] = identifier = name + tident = nimState.newTypeIdent(node[i]) + + if not tident.isNil: + let + # Only need the ident tree, not nkTypeDef parent + ident = tident[0] + name = tident.getIdentName() - # node[start+1] could have nested pointers - tcount = node[start+1].getPtrCount() + # node[i] could have nested pointers + tcount = node[i].getPtrCount() - # node[start] = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + # node[start] = identifier = return type name + (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) - # Parameter list - plist = node[start+1].anyChildInTree("parameter_list") + # Parameter list + plist = node[i].anyChildInTree("parameter_list") - procDef = newNode(nkProcDef) + procDef = newNode(nkProcDef) - # proc X(a1: Y, a2: Z): P {.pragma.} - # - # nkProcDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkEmpty(), - # nkFormalParams( - # nkPtrTy( # optional, nested - # nkIdent(retType) - # ), - # nkIdentDefs( - # nkIdent(param), - # nkPtrTy( - # nkIdent(ptype) - # ), - # nkEmpty() - # ), - # ... - # ), - # nkPragma(...), - # nkEmpty(), - # nkEmpty() - # ) + # proc X(a1: Y, a2: Z): P {.pragma.} + # + # nkProcDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkEmpty(), + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkPragma(...), + # nkEmpty(), + # nkEmpty() + # ) - procDef.add ident - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) + procDef.add ident + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) - # Return type - var - retType = nimState.getIdent(rname, rinfo, exported = false) - if tcount > 0: - retType = nimState.newPtrTree(tcount, retType) + # Return type + var + retType = + if rname == "object" and tcount == 0: + # void func(..) + newNode(nkEmpty) + else: + nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = nimState.newPtrTree(tcount, retType) - # Proc with return type and params - procDef.add nimState.newFormalParams(name, plist, retType) - procDef.add newNode(nkEmpty) # Pragmas - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) + # Proc with return type and params + procDef.add nimState.newFormalParams(name, plist, retType) + procDef.add newNode(nkEmpty) # Pragmas + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) - # nkProcSection.add - nimState.procSection.add procDef + # nkProcSection.add + nimState.procSection.add procDef - nimState.printDebug(procDef) + nimState.printDebug(procDef) proc processNode(nimState: NimState, node: TSNode): bool = result = true @@ -1247,6 +1291,8 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = nimState.searchTree(root) + nimState.addAllOverrideFinal() + var tree = newNode(nkStmtList) tree.add nimState.pragmaSection diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 251955d..044cde7 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -174,6 +174,7 @@ proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool # Overrides related proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = + # Get cOverride for identifier `name` of `kind` if defined doAssert name.nBl, "Blank identifier error" if nimState.gState.onSymbolOverride != nil: @@ -190,6 +191,7 @@ proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = result = result.replace(re"(?m)^(.*?)$", " $1") proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = + # Get all unused cOverride symbols of `kind` let typ = $kind @@ -197,6 +199,11 @@ proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = for i in nimState.gState.onSymbolOverrideFinal(typ): result &= "\n" & nimState.getOverride(i, kind) +proc getKeyword*(kind: NimSymKind): string = + # Convert `kind` into a Nim keyword + # cOverride procs already include `proc` keyword + result = ($kind).replace("nsk", "").toLowerAscii() + # TSNode shortcuts proc isNil*(node: TSNode): bool = diff --git a/tests/include/tast2.h b/tests/include/tast2.h index b867020..84505db 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -25,7 +25,7 @@ typedef char *(*A11)[3]; typedef struct A1 *A111[12]; typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); -typedef int A13(int, int); +typedef int A13(int, int, void (*func)(void)); struct A14 { volatile char a1; }; struct A15 { char *a1; const int *a2[1]; }; diff --git a/tests/tast2.nim b/tests/tast2.nim index 914d40f..56b1868 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -54,7 +54,7 @@ assert A11 is ptr array[3, cstring] assert A111 is array[12, ptr A1] assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint -assert A13 is proc(a1: cint, a2: cint): cint +assert A13 is proc(a1: cint, a2: cint, `func`: proc()): cint assert A14 is object testFields(A14, {"a1": "cchar"}.toTable()) From e98564528ee8a9fc934bb2de31114b66673b3eaa Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 24 Mar 2020 15:48:54 -0500 Subject: [PATCH 017/255] ast2 pragmas for types --- nimterop/ast2.nim | 31 ++++++++++++++++++++++++------- nimterop/build.nim | 2 +- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index d3d32c3..62fdcf2 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -192,8 +192,13 @@ proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, va colExpr.add value pragma.add colExpr +proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: seq[string]) = + # Add sequence of pragmas to an existing nkPragma tree + for name in pragmas: + nimState.addPragma(node, pragma, name) + proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: OrderedTable[string, PNode]) = - # Add pragmas to an existing nkPragma tree + # Add a table of name:value pragmas to an existing nkPragma tree for name, value in pragmas.pairs: nimState.addPragma(node, pragma, name, value) @@ -212,7 +217,7 @@ proc newPragma(nimState: NimState, node: TSNode, name: string, value: PNode = ni result = newNode(nkPragma) nimState.addPragma(node, result, name, value) -proc newPragma(nimState: NimState, node: TSNode, pragmas: OrderedTable[string, PNode]): PNode = +proc newPragma(nimState: NimState, node: TSNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = # Create nkPragma tree for multiple name:value result = newNode(nkPragma) nimState.addPragma(node, result, pragmas) @@ -237,13 +242,13 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, name: string, result.add ident result.add nimState.newPragma(node, name, value) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: OrderedTable[string, PNode]): PNode {.used.} = +proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = # Create nkPragmaExpr tree for multiple name:value result = newNode(nkPragmaExpr) result.add ident result.add nimState.newPragma(node, pragmas) -proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): PNode = +proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", pragmas: seq[string] = @[]): PNode = # Create nkTypeDef PNode with first ident # # If `fname`, use it instead of node.getAtom() for name @@ -258,8 +263,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", union = false): ident = nimState.getIdent(name, info) prident = - if union: - nimState.newPragmaExpr(node, ident, "union") + if pragmas.nBl and not ident.isNil: + nimState.newPragmaExpr(node, ident, pragmas) else: ident @@ -509,9 +514,15 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname # If `fname` is set, use it as the name when creating new PNode decho("addTypeObject()") let + pragmas = + if union: + @["union", "bycopy"] + else: + @["bycopy"] + typeDef = if typeDef.isNil: - nimState.newTypeIdent(node, fname, union = union) + nimState.newTypeIdent(node, fname, pragmas) else: typeDef @@ -546,6 +557,12 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname typeDef.add obj + # If typeDef was passed in, need to add pragmas if any + if pragmas.nBl and typeDef[0].kind != nkPragmaExpr: + let + npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) + typedef[0] = npexpr + # nkTypeSection.add nimState.typeSection.add typeDef diff --git a/nimterop/build.nim b/nimterop/build.nim index 7939afa..9594baa 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -958,7 +958,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta preBuild = newIdentNode(name & "PreBuild") # Regex for library search - lre = "(lib)?$1[_]?(static)?[0-9.\\-]*\\" + lre = "(lib)?$1[_-]?(static)?[0-9.\\-]*\\" # If -d:xxx set with setDefines() stdVal = gDefines.hasKey(stdStr) From 8b9c39f42eec79607392c81538e08da1b814809a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 25 Mar 2020 15:17:20 -0500 Subject: [PATCH 018/255] ast2 newTypeIdent cleanup, pragma tests, docs.nim docs --- nimterop.nimble | 1 + nimterop/all.nim | 2 +- nimterop/ast2.nim | 91 +++++++++++++++++++++++++++---------------- nimterop/docs.nim | 4 ++ tests/include/tast2.h | 12 ++++-- tests/tast2.nim | 85 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 154 insertions(+), 41 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index c33e78b..86ee029 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -37,6 +37,7 @@ task test, "Test": buildToastTask() execTest "tests/tast2.nim" + #execCmd "nim c -f -d:HEADER -r tests/tast2.nim" execTest "tests/tnimterop_c.nim" execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" diff --git a/nimterop/all.nim b/nimterop/all.nim index dc22967..fef1bdd 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -4,4 +4,4 @@ Module that should import everything so that `nim doc --project nimtero/all` run # TODO: make sure it does import everything. -import "."/[cimport, build, types, plugin] +import "."/[docs, cimport, build, types, plugin] diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 62fdcf2..93b26a1 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -248,12 +248,13 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: seq[ result.add ident result.add nimState.newPragma(node, pragmas) -proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", pragmas: seq[string] = @[]): PNode = - # Create nkTypeDef PNode with first ident +proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[]): PNode = + # Create nkTypeDef PNode with first ident if `nskType` else just create an nkPostfix node for `nskProc` # # If `fname`, use it instead of node.getAtom() for name + # If `pragmas`, add as nkPragmaExpr but only if `nskType` since procs add pragmas elsewhere let - (tname, origname, info) = nimState.getNameInfo(node.getAtom(), nskType) + (tname, origname, info) = nimState.getNameInfo(node.getAtom(), kind) name = if fname.nBl: @@ -262,32 +263,49 @@ proc newTypeIdent(nimState: NimState, node: TSNode, fname = "", pragmas: seq[str tname ident = nimState.getIdent(name, info) - prident = - if pragmas.nBl and not ident.isNil: - nimState.newPragmaExpr(node, ident, pragmas) - else: - ident - if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, nskType) + result = nimState.getOverrideOrSkip(node, origname, kind) elif nimState.addNewIdentifer(name): - # type name* = - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent(name) - # ), - # nkEmpty() - # ) - result = newNode(nkTypeDef) - result.add prident - result.add newNode(nkEmpty) + if kind == nskType: + # type name* = + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkEmpty() + # ) + let + pragmas = + if nimState.includeHeader and tname == origname: + # Need to add impShort + pragmas & nimState.impShort + else: + pragmas + + prident = + if pragmas.nBl and not ident.isNil: + nimState.newPragmaExpr(node, ident, pragmas) + else: + ident + + result = newNode(nkTypeDef) + result.add prident + result.add newNode(nkEmpty) + elif kind == nskProc: + # name* + # + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ) + result = ident nimState.identifierNodes[name] = result else: - necho &"# type '{origname}' is duplicate, skipped" + necho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count @@ -520,9 +538,11 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname else: @["bycopy"] + typeDefExisting = not typeDef.isNil + typeDef = if typeDef.isNil: - nimState.newTypeIdent(node, fname, pragmas) + nimState.newTypeIdent(node, fname = fname, pragmas = pragmas) else: typeDef @@ -558,10 +578,14 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname typeDef.add obj # If typeDef was passed in, need to add pragmas if any - if pragmas.nBl and typeDef[0].kind != nkPragmaExpr: - let - npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) - typedef[0] = npexpr + if pragmas.nBl and typeDefExisting: + if typeDef[0].kind != nkPragmaExpr: + # includeHeader already added impShort + let + npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) + typedef[0] = npexpr + else: + nimState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add nimState.typeSection.add typeDef @@ -1112,13 +1136,12 @@ proc addProc(nimState: NimState, node: TSNode) = for i in start+1 ..< node.len: let # node[i] = identifier = name - tident = nimState.newTypeIdent(node[i]) + ident = nimState.newTypeIdent(node[i], kind = nskProc) - if not tident.isNil: + if not ident.isNil: let # Only need the ident tree, not nkTypeDef parent - ident = tident[0] - name = tident.getIdentName() + name = ident.getIdentName() # node[i] could have nested pointers tcount = node[i].getPtrCount() @@ -1258,7 +1281,7 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = nimState.constSection.add nimState.newConstDef( root, fname = nimState.currentHeader, fval = '"' & fullpath & '"') - nimState.addPragma(root, impPragma, "header", newStrNode(nkStrLit, nimState.currentHeader)) + nimState.addPragma(root, impPragma, "header", nimState.getIdent(nimState.currentHeader)) nimState.addPragma(root, impCPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) nimState.addPragma(root, impCPragma, nimState.impShort) @@ -1312,9 +1335,9 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = var tree = newNode(nkStmtList) - tree.add nimState.pragmaSection tree.add nimState.enumSection tree.add nimState.constSection + tree.add nimState.pragmaSection tree.add nimState.typeSection tree.add nimState.procSection diff --git a/nimterop/docs.nim b/nimterop/docs.nim index a194086..97fab96 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -1,6 +1,10 @@ import macros, strformat from os import parentDir, getCurrentCompilerExe, DirSep + +when defined(nimdoc): + from os import getCurrentDir, paramCount, paramStr + proc getNimRootDir(): string = #[ hack, but works diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 84505db..8e84b01 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -11,7 +11,7 @@ typedef struct A3 {}; typedef struct A4 A4, *A4p; typedef const int A5; typedef int *A6; -typedef A0 **A7; +typedef struct A0 **A7; typedef void *A8; // Forward declaration @@ -104,6 +104,8 @@ typedef enum VSPresetFormat { // DUPLICATES +#ifndef HEADER + #define A 1 #define B 1.0 #define C 0x10 @@ -117,7 +119,7 @@ typedef struct A3 {}; typedef struct A4 A4, *A4p; typedef const int A5; typedef int *A6; -typedef A0 **A7; +typedef struct A0 **A7; typedef void *A8; // Forward declaration @@ -131,7 +133,7 @@ typedef char *(*A11)[3]; typedef struct A1 *A111[12]; typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); -typedef int A13(int, int); +typedef int A13(int, int, void (*func)(void)); struct A14 { volatile char a1; }; struct A15 { char *a1; const int *a2[1]; }; @@ -200,4 +202,6 @@ typedef enum VSPresetFormat { // Anonymous //typedef struct { char a1; }; -//struct A2 test_proc1(struct A0 a); \ No newline at end of file +//struct A2 test_proc1(struct A0 a); + +#endif \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index 56b1868..76e4561 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -1,4 +1,4 @@ -import tables +import macros, sets, tables import nimterop/[cimport] @@ -12,7 +12,36 @@ cOverride: type A1* = A0 -cImport("include/tast2.h", flags="-d -f:ast2 -ENK_") +when defined(HEADER): + cDefine("HEADER") + const + flags = " -H" + imp = @["importc", "header:headertast2"] +else: + const + flags = "" + imp = @[] + +cImport("include/tast2.h", flags="-d -f:ast2 -ENK_" & flags) + +proc getPragmas(n: NimNode): HashSet[string] = + for i in 0 ..< n.len: + if n[i].kind == nnkPragma: + for j in 0 ..< n[i].len: + if n[i][j].kind == nnkIdent: + result.incl $n[i][j] + elif n[i][j].kind == nnkExprColonExpr: + result.incl $n[i][j][0] & ":" & $n[i][j][1] + else: + result.incl n[i].getPragmas() + +macro checkPragmas(t: typed, pragmas: static[seq[string]]): untyped = + let + ast = t.getImpl() + prag = ast.getPragmas() + exprag = pragmas.toHashSet() + doAssert symmetricDifference(prag, exprag).len == 0, + "\nWrong number of pragmas in " & $t & "\n" & $prag & " vs " & $exprag proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = var @@ -31,62 +60,114 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +const + pragmas = @["bycopy"] & imp + assert A0 is object testFields(A0, {"f1": "cint"}.toTable()) +checkPragmas(A0, pragmas) + assert A1 is A0 testFields(A1, {"f1": "cint"}.toTable()) + assert A2 is object testFields(A2) +checkPragmas(A2, pragmas) + assert A3 is object testFields(A3) +checkPragmas(A3, pragmas) + assert A4 is object testFields(A4) +checkPragmas(A4, pragmas) + assert A4p is ptr A4 +checkPragmas(A4p, imp) + assert A5 is cint +checkPragmas(A5, imp) + assert A6 is ptr cint +checkPragmas(A6, imp) + assert A7 is ptr ptr A0 +checkPragmas(A7, imp) + assert A8 is pointer +checkPragmas(A8, imp) assert A9p is array[3, cstring] +checkPragmas(A9p, imp) + #assert A9 is array[4, cchar] +#checkPragmas(A9, imp) + assert A10 is array[3, array[6, cstring]] +checkPragmas(A10, imp) + assert A11 is ptr array[3, cstring] +checkPragmas(A11, imp) + assert A111 is array[12, ptr A1] +checkPragmas(A111, imp) assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint +checkPragmas(A12, imp) + assert A13 is proc(a1: cint, a2: cint, `func`: proc()): cint +checkPragmas(A13, imp) assert A14 is object testFields(A14, {"a1": "cchar"}.toTable()) +checkPragmas(A14, pragmas) assert A15 is object testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +checkPragmas(A15, pragmas) assert A16 is object testFields(A16, {"f1": "cchar"}.toTable()) +checkPragmas(A16, pragmas) assert A17 is object testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +checkPragmas(A17, pragmas) + assert A18 is A17 +checkPragmas(A18, imp) + assert A18p is ptr A17 +checkPragmas(A18p, imp) assert A19 is object testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +checkPragmas(A19, pragmas) + assert A19p is ptr A19 +checkPragmas(A19p, imp) assert A20 is object testFields(A20, {"a1": "cchar"}.toTable()) +checkPragmas(A20, pragmas) + assert A21 is A20 +checkPragmas(A21, imp) + assert A21p is ptr A20 +checkPragmas(A21p, imp) assert A22 is object testFields(A22, {"f1": "ptr ptr cint", "f2": "array[0..254, ptr cint]"}.toTable()) +checkPragmas(A22, pragmas) assert U1 is object assert sizeof(U1) == sizeof(cfloat) +checkPragmas(U1, pragmas & @["union"]) assert U2 is object assert sizeof(U2) == 256 * sizeof(cint) +checkPragmas(U2, pragmas & @["union"]) assert PANEL_WINDOW == 1 assert PANEL_GROUP == 2 From e06917f40f5bd3a4b4bec51d05d85dffa4eb5867 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 30 Mar 2020 15:46:34 -0500 Subject: [PATCH 019/255] c2nImport ret, del specials from header, cache if exec not die, ast2 type pragma improvements --- nimterop.nimble | 5 +- nimterop/ast2.nim | 138 +++++++++++++++++------ nimterop/build.nim | 9 +- nimterop/cimport.nim | 2 + tests/include/tast2.h | 8 ++ tests/tast2.nim | 249 +++++++++++++++++++++++++++++++----------- 6 files changed, 306 insertions(+), 105 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 86ee029..108d935 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -5,8 +5,6 @@ author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" -# this gives Warning: Binary 'nimterop/toast' was already installed from source directory -# when running `nimble install --verbose -y` bin = @["nimterop/toast"] installDirs = @["nimterop"] installFiles = @["config.nims"] @@ -37,7 +35,7 @@ task test, "Test": buildToastTask() execTest "tests/tast2.nim" - #execCmd "nim c -f -d:HEADER -r tests/tast2.nim" + execCmd "nim c -f -d:HEADER -r tests/tast2.nim" execTest "tests/tnimterop_c.nim" execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" @@ -53,5 +51,6 @@ task test, "Test": # getHeader tests withDir("tests"): execCmd("nim e getheader.nims") + execCmd("nim e wrappers.nims") docsTask() diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 93b26a1..7eb22a1 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -57,7 +57,7 @@ proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: let # Get cleaned name for symbol, set parent so that cOverride is ignored name = nimState.getIdentifier(origname, kind, parent = "override") - + override = nimState.getOverride(origname, kind) var @@ -115,7 +115,7 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode else: # node[0] = identifier = const name nimState.getNodeVal(node.getAtom()) - + name = nimState.getIdentifier(origname, nskConst) info = nimState.getLineInfo(node) ident = nimState.getIdent(name, info) @@ -182,7 +182,7 @@ proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, va let pinfo = nimState.getLineInfo(node.getAtom()) pident = nimState.getIdent(name, pinfo, exported = false) - + if value.isNil: pragma.add pident else: @@ -248,19 +248,28 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: seq[ result.add ident result.add nimState.newPragma(node, pragmas) -proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[]): PNode = +proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[], istype = false): PNode = # Create nkTypeDef PNode with first ident if `nskType` else just create an nkPostfix node for `nskProc` # # If `fname`, use it instead of node.getAtom() for name # If `pragmas`, add as nkPragmaExpr but only if `nskType` since procs add pragmas elsewhere + # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} when includeHeader let - (tname, origname, info) = nimState.getNameInfo(node.getAtom(), kind) + (tname, torigname, info) = nimState.getNameInfo(node.getAtom(), kind) - name = + origname = if fname.nBl: fname + else: + torigname + + # Process name if forced, getNameInfo() already runs getIdentifier() + name = + if fname.nBl: + nimState.getIdentifier(fname, kind) else: tname + ident = nimState.getIdent(name, info) if name.Bl: @@ -277,20 +286,39 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", # ), # nkEmpty() # ) - let + var pragmas = - if nimState.includeHeader and tname == origname: - # Need to add impShort - pragmas & nimState.impShort + if nimState.includeHeader: + # Need to add header and importc + if istype and name == origname: + # Need to add impShort since neither struct/union nor name change + pragmas & nimState.impShort + else: + # Add header shortcut, additional pragmas added later + pragmas & (nimState.impShort & "H") else: pragmas prident = - if pragmas.nBl and not ident.isNil: + if pragmas.nBl: nimState.newPragmaExpr(node, ident, pragmas) else: ident - + + if nimState.includeHeader: + if not istype or name != origname: + # Add importc pragma since either struct/union or name changed + let + uors = + if not istype: + if "union" in pragmas: + "union " + else: + "struct " + else: + "" + nimState.addPragma(node, prident[1], "importc", newStrNode(nkStrLit, &"{uors}{origname}")) + result = newNode(nkTypeDef) result.add prident result.add newNode(nkEmpty) @@ -302,7 +330,7 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", # nkIdent(name) # ) result = ident - + nimState.identifierNodes[name] = result else: necho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) @@ -525,24 +553,37 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = if not field.isNil: result.add field -proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", union = false) = +proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object # # If `typeDef` is set, use it instead of creating new PNode # If `fname` is set, use it as the name when creating new PNode + # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") let - pragmas = - if union: - @["union", "bycopy"] + # Object has fields or not + fdlist = node.anyChildInTree("field_declaration_list") + + pragmas = block: + var pragmas = + if union: + @["union"] + else: + @[] + if not fdlist.isNil and fdlist.len > 0: + # Object with fields should be bycopy + pragmas.add "bycopy" else: - @["bycopy"] + # Incomplete, might get forward declared + pragmas.add "incompleteStruct" + + pragmas typeDefExisting = not typeDef.isNil typeDef = if typeDef.isNil: - nimState.newTypeIdent(node, fname = fname, pragmas = pragmas) + nimState.newTypeIdent(node, fname = fname, pragmas = pragmas, istype = istype) else: typeDef @@ -567,8 +608,6 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname obj.add newNode(nkEmpty) obj.add newNode(nkEmpty) - let - fdlist = node.anyChildInTree("field_declaration_list") if not fdlist.isNil and fdlist.len > 0: # Add fields to object if present obj.add nimState.newRecListTree(name, fdlist) @@ -580,11 +619,11 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname # If typeDef was passed in, need to add pragmas if any if pragmas.nBl and typeDefExisting: if typeDef[0].kind != nkPragmaExpr: - # includeHeader already added impShort let npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) typedef[0] = npexpr else: + # includeHeader already added impShort in newTypeIdent() nimState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add @@ -610,6 +649,18 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname # Add fields to existing object def[2][2] = nimState.newRecListTree(name, fdlist) + # Change incompleteStruct to bycopy pragma + if def[0].kind == nkPragmaExpr and def[0].len == 2 and + def[0][1].kind == nkPragma and def[0][1].len > 0: + for i in 0 ..< def[0][1].len: + if $def[0][1][i] == "incompleteStruct": + def[0][1][i] = nimState.getIdent( + "bycopy", nimState.getLineInfo(node.getAtom()), + exported = false + ) + + nimState.printDebug(def) + proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specified type # @@ -622,8 +673,8 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specific type let # node[i] = identifer = name - typeDef = nimState.newTypeIdent(node[i]) - + typeDef = nimState.newTypeIdent(node[i], istype = true) + if not typeDef.isNil: let name = typeDef.getIdentName() @@ -669,7 +720,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = nimState.printDebug(typeDef) else: - nimState.addTypeObject(node, typeDef = typeDef) + nimState.addTypeObject(node, typeDef = typeDef, istype = true) proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode = # Create array type tree @@ -728,7 +779,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = decho("addTypeArray()") let # node[1] = identifer = name - typeDef = nimState.newTypeIdent(node[1]) + typeDef = nimState.newTypeIdent(node[1], istype = true) if not typeDef.isNil: let @@ -809,7 +860,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = decho("addTypeProc()") let # node[1] = identifier = name - typeDef = nimState.newTypeIdent(node[1]) + typeDef = nimState.newTypeIdent(node[1], istype = true) if not typeDef.isNil: let @@ -1018,7 +1069,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # First add struct as object decho("addType(): case 6") - nimState.addTypeObject(node[0], union = union) + nimState.addTypeObject(node[0], istype = true, union = union) if node.len > 1 and nimState.getNodeVal(node[1]) != "": # Add any additional names @@ -1041,7 +1092,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = name # Now add struct as object with specified name - nimState.addTypeObject(node[0], fname = name, union = union) + nimState.addTypeObject(node[0], fname = name, istype = true, union = union) if name.nBl: # Add any additional names @@ -1068,7 +1119,7 @@ proc addEnum(nimState: NimState, node: TSNode) = # Use Y as name origname = nimState.getNodeVal(node[1].getAtom()) offset = 1 - + if origname.nBl: name = nimState.getIdentifier(origname, nskType) else: @@ -1120,7 +1171,7 @@ proc addEnum(nimState: NimState, node: TSNode) = # Add fields to list of consts after processing enum so that we don't cast # enum field to itself nimState.constIdentifiers.incl fnames - + # Add other names if node.getName() == "type_definition" and node.len > 1: nimState.addTypeTyped(node, ftname = name, offset = offset) @@ -1137,7 +1188,7 @@ proc addProc(nimState: NimState, node: TSNode) = let # node[i] = identifier = name ident = nimState.newTypeIdent(node[i], kind = nskProc) - + if not ident.isNil: let # Only need the ident tree, not nkTypeDef parent @@ -1270,10 +1321,13 @@ proc searchTree(nimState: NimState, root: TSNode) = break proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = + # Create shortcut pragmas to reduce clutter var + hdrPragma: PNode impPragma = newNode(nkPragma) impCPragma = newNode(nkPragma) + # {.importc.} nimState.addPragma(root, impPragma, "pragma", nimState.getIdent(nimState.impShort)) nimState.addPragma(root, impPragma, "importc") @@ -1281,19 +1335,29 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = nimState.constSection.add nimState.newConstDef( root, fname = nimState.currentHeader, fval = '"' & fullpath & '"') - nimState.addPragma(root, impPragma, "header", nimState.getIdent(nimState.currentHeader)) + # {.header: "xxx".} + hdrPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "H")) + nimState.addPragma(root, hdrPragma, "header", nimState.getIdent(nimState.currentHeader)) + nimState.addPragma(root, impPragma, nimState.impShort & "H") + + # {.importc.} + {.cdecl.} for procs nimState.addPragma(root, impCPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) nimState.addPragma(root, impCPragma, nimState.impShort) nimState.addPragma(root, impCPragma, "cdecl") if nimState.gState.dynlib.nBl: + # {.dynlib.} for DLLs nimState.addPragma(root, impCPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + # Add all pragma shortcuts to output + if not hdrPragma.isNil: + nimState.pragmaSection.add hdrPragma nimState.pragmaSection.add impPragma nimState.pragmaSection.add impCPragma proc printNimHeader*(gState: State) = + # Top level output with context info gecho """# Generated at $1 # Command line: # $2 $3 @@ -1304,13 +1368,16 @@ import nimterop/types """ % [$now(), getAppFilename(), commandLineParams().join(" ")] proc printNim*(gState: State, fullpath: string, root: TSNode) = + # Generate Nim from tree-sitter AST root node let nimState = new(NimState) fp = fullpath.replace("\\", "/") + # Track identifiers already rendered and corresponding PNodes nimState.identifiers = newTable[string, string]() nimState.identifierNodes = newTable[string, PNode]() + # toast objects nimState.gState = gState nimState.currentHeader = getCurrentHeader(fullpath) nimState.impShort = nimState.currentHeader.replace("header", "imp") @@ -1321,18 +1388,23 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = nimState.config = newConfigRef() nimstate.graph = newModuleGraph(nimState.identCache, nimState.config) + # Initialize all section PNodes nimState.pragmaSection = newNode(nkStmtList) nimState.constSection = newNode(nkConstSection) nimState.enumSection = newNode(nkStmtList) nimState.procSection = newNode(nkStmtList) nimState.typeSection = newNode(nkTypeSection) + # Setup pragmas nimState.setupPragmas(root, fp) + # Search root node and render Nim nimState.searchTree(root) + # Add any unused cOverride symbols to output nimState.addAllOverrideFinal() + # Create output to Nim using Nim compiler renderer var tree = newNode(nkStmtList) tree.add nimState.enumSection diff --git a/nimterop/build.nim b/nimterop/build.nim index 9594baa..c65ec2a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep +import regex + proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: @@ -83,7 +85,7 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, else: # Execute command and store results in cache (result.output, result.ret) = gorgeEx(ccmd) - if result.ret == 0: + if result.ret == 0 or die == false: # mkdir for execCache dir (circular dependency) let dir = cacheFile.parentDir() if not dirExists(dir): @@ -933,7 +935,8 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called ## prior to the build process. var - name = header.extractFilename().split(".")[0] + origname = header.extractFilename().split(".")[0] + name = origname.replace(re"[[:^alnum:]]", "") # -d:xxx for this header stdStr = name & "Std" @@ -975,7 +978,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta if altNames.len != 0: lre = lre % ("(" & altNames.replace(",", "|") & ")") else: - lre = lre % name + lre = lre % origname result = newNimNode(nnkStmtList) result.add(quote do: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 84f7b4f..c4ebc3f 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -673,6 +673,8 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: (c2nimout, ret) = execAction(cmd, cache = not gStateCT.nocache, cacheKey = getCacheValue(hpath)) + doAssert ret == 0, "\n\nc2nim codegen limitation or error - " & c2nimout + var nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 8e84b01..db539c7 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -19,6 +19,10 @@ struct A0 { int f1; }; +struct A4 { + float f1; +}; + typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; @@ -127,6 +131,10 @@ struct A0 { int f1; }; +struct A4 { + float f1; +}; + typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; diff --git a/tests/tast2.nim b/tests/tast2.nim index 76e4561..4b19429 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -1,10 +1,30 @@ -import macros, sets, tables +import macros, os, sets, strutils import nimterop/[cimport] static: cDebug() +const + path = currentSourcePath.parentDir() / "include" / "tast2.h" + +when defined(HEADER): + cDefine("HEADER") + const + flags = " -H -d" + pHeader = @["header:" & path] + pHeaderImp = @["importc"] & pHeader +else: + const + flags = "" + pHeader: seq[string] = @[] + pHeaderImp: seq[string] = @[] + +const + pHeaderImpBy = @["bycopy"] & pHeaderImp + pHeaderBy = @["bycopy"] & pHeader + pHeaderInc = @["incompleteStruct"] & pHeader + cOverride: const A* = 2 @@ -12,19 +32,10 @@ cOverride: type A1* = A0 -when defined(HEADER): - cDefine("HEADER") - const - flags = " -H" - imp = @["importc", "header:headertast2"] -else: - const - flags = "" - imp = @[] - -cImport("include/tast2.h", flags="-d -f:ast2 -ENK_" & flags) +cImport(path, flags="-f:ast2 -ENK_" & flags) proc getPragmas(n: NimNode): HashSet[string] = + # Find all pragmas in AST, return as "name" or "name:value" in set for i in 0 ..< n.len: if n[i].kind == nnkPragma: for j in 0 ..< n[i].len: @@ -35,24 +46,54 @@ proc getPragmas(n: NimNode): HashSet[string] = else: result.incl n[i].getPragmas() -macro checkPragmas(t: typed, pragmas: static[seq[string]]): untyped = - let +proc getRecList(n: NimNode): NimNode = + # Find nnkRecList in AST + for i in 0 ..< n.len: + if n[i].kind == nnkRecList: + return n[i] + elif n[i].len != 0: + let + rl = getRecList(n[i]) + if not rl.isNil: + return rl + +macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] = true): untyped = + # Verify that type has expected pragmas defined + # `istype` is true when typedef X + var ast = t.getImpl() prag = ast.getPragmas() exprag = pragmas.toHashSet() + when defined(HEADER): + if not istype: + if "union" in exprag: + exprag.incl "importc:union " & $t + else: + exprag.incl "importc:struct " & $t doAssert symmetricDifference(prag, exprag).len == 0, "\nWrong number of pragmas in " & $t & "\n" & $prag & " vs " & $exprag -proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = +macro testFields(t: typed, fields: static[string] = "") = + # Verify that type has expected fields var - obj: t - count = 0 - for name, value in obj.fieldPairs(): - count += 1 - assert name in fields, $t & "." & name & " invalid" - assert $fields[name] == $typeof(value), - "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value) - assert count == fields.len, "Failed for " & $t + ast = t.getImpl() + rl = ast.getRecList() + fsplit = fields.split(":") + names = fsplit[0].split("|") + types = + if fsplit.len > 1: + fsplit[1].split("|") + else: + @[] + if not rl.isNil: + for i in 0 ..< rl.len: + let + name = ($rl[i][0]).strip(chars = {'*'}) + typ = $(rl[i][1].repr()) + n = names.find(name) + assert n != -1, $t & "." & name & " invalid" + assert types[n] == typ, + "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ assert A == 2 assert B == 1.0 @@ -60,114 +101,190 @@ assert C == 0x10 assert D == "hello" assert E == 'c' -const - pragmas = @["bycopy"] & imp - assert A0 is object -testFields(A0, {"f1": "cint"}.toTable()) -checkPragmas(A0, pragmas) +testFields(A0, "f1:cint") +checkPragmas(A0, pHeaderBy, istype = false) +var a0: A0 +a0.f1 = 1 assert A1 is A0 -testFields(A1, {"f1": "cint"}.toTable()) +testFields(A1, "f1:cint") +var a1: A1 +a1.f1 = 2 assert A2 is object testFields(A2) -checkPragmas(A2, pragmas) +checkPragmas(A2, pHeaderInc, istype = false) +when not defined(HEADER): + # typedef struct X; is invalid + var a2: A2 assert A3 is object testFields(A3) -checkPragmas(A3, pragmas) +checkPragmas(A3, pHeaderInc, istype = false) +var a3: A3 assert A4 is object -testFields(A4) -checkPragmas(A4, pragmas) +testFields(A4, "f1:cfloat") +checkPragmas(A4, pHeaderImpBy) +var a4: A4 +a4.f1 = 4.1 assert A4p is ptr A4 -checkPragmas(A4p, imp) +testFields(A4p, "f1:cfloat") +checkPragmas(A4p, pHeaderImp) +var a4p: A4p +a4p = addr a4 assert A5 is cint -checkPragmas(A5, imp) +checkPragmas(A5, pHeaderImp) +const a5: A5 = 5 assert A6 is ptr cint -checkPragmas(A6, imp) +checkPragmas(A6, pHeaderImp) +var + a6: A6 + a6i = 6 +a6 = cast[A6](addr a6i) assert A7 is ptr ptr A0 -checkPragmas(A7, imp) +checkPragmas(A7, pHeaderImp) +var + a7: A7 + a7a = addr a0 +a7 = addr a7a assert A8 is pointer -checkPragmas(A8, imp) +checkPragmas(A8, pHeaderImp) +var a8: A8 +a8 = nil assert A9p is array[3, cstring] -checkPragmas(A9p, imp) +checkPragmas(A9p, pHeaderImp) +var a9p: A9p +a9p[1] = nil +a9p[2] = "hello".cstring +#Not implemented yet #assert A9 is array[4, cchar] -#checkPragmas(A9, imp) +#checkPragmas(A9, pHeaderImp) +#var a9: A9 assert A10 is array[3, array[6, cstring]] -checkPragmas(A10, imp) +checkPragmas(A10, pHeaderImp) +var a10: A10 +a10[2][5] = "12345".cstring assert A11 is ptr array[3, cstring] -checkPragmas(A11, imp) +checkPragmas(A11, pHeaderImp) +var a11: A11 +a11 = addr a9p assert A111 is array[12, ptr A1] -checkPragmas(A111, imp) +checkPragmas(A111, pHeaderImp) +var a111: A111 +a111[11] = addr a1 assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint -checkPragmas(A12, imp) +checkPragmas(A12, pHeaderImp) +when not defined(HEADER): + # Unclear why this fails + # request for member ‘ClE_0’ in something not a structure or union + var a12: A12 assert A13 is proc(a1: cint, a2: cint, `func`: proc()): cint -checkPragmas(A13, imp) +checkPragmas(A13, pHeaderImp) +when not defined(HEADER): + # Unclear why this fails + # request for member ‘ClE_0’ in something not a structure or union + var a13: A13 assert A14 is object -testFields(A14, {"a1": "cchar"}.toTable()) -checkPragmas(A14, pragmas) +testFields(A14, "a1:cchar") +checkPragmas(A14, pHeaderBy, istype = false) +var a14: A14 +a14.a1 = 'a' assert A15 is object -testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) -checkPragmas(A15, pragmas) +testFields(A15, "a1|a2:cstring|array[1, ptr cint]") +checkPragmas(A15, pHeaderBy, istype = false) +var + a15: A15 + a15i = 15.cint +a15.a1 = "hello".cstring +a15.a2[0] = addr a15i assert A16 is object -testFields(A16, {"f1": "cchar"}.toTable()) -checkPragmas(A16, pragmas) +testFields(A16, "f1:cchar") +checkPragmas(A16, pHeaderImpBy) +when not defined(HEADER): + # Similar to A2 + var a16: A16 + a16.f1 = 's' assert A17 is object -testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) -checkPragmas(A17, pragmas) +testFields(A17, "a1|a2:cstring|array[1, ptr cint]") +checkPragmas(A17, pHeaderImpBy) +when not defined(HEADER): + # Similar to A2 + var a17: A17 + a17.a1 = "hello".cstring + a17.a2[0] = addr a15i assert A18 is A17 -checkPragmas(A18, imp) +checkPragmas(A18, pHeaderImp) +var a18: A18 assert A18p is ptr A17 -checkPragmas(A18p, imp) +checkPragmas(A18p, pHeaderImp) +var a18p: A18p +a18p = addr a18 assert A19 is object -testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) -checkPragmas(A19, pragmas) +testFields(A19, "a1|a2:cstring|array[1, ptr cint]") +checkPragmas(A19, pHeaderImpBy) +var a19: A19 +a19.a1 = "hello".cstring +a19.a2[0] = addr a15i assert A19p is ptr A19 -checkPragmas(A19p, imp) +checkPragmas(A19p, pHeaderImp) +var a19p: A19p +a19p = addr a19 assert A20 is object -testFields(A20, {"a1": "cchar"}.toTable()) -checkPragmas(A20, pragmas) +testFields(A20, "a1:cchar") +checkPragmas(A20, pHeaderImpBy) +var a20: A20 +a20.a1 = 'a' assert A21 is A20 -checkPragmas(A21, imp) +checkPragmas(A21, pHeaderImp) +var a21: A21 +a21 = a20 assert A21p is ptr A20 -checkPragmas(A21p, imp) +checkPragmas(A21p, pHeaderImp) +var a21p: A21p +a21p = addr a20 assert A22 is object -testFields(A22, {"f1": "ptr ptr cint", "f2": "array[0..254, ptr cint]"}.toTable()) -checkPragmas(A22, pragmas) +testFields(A22, "f1|f2:ptr ptr cint|array[123 + 132, ptr cint]") +checkPragmas(A22, pHeaderImpBy) +var a22: A22 +a22.f1 = addr a15.a2[0] assert U1 is object assert sizeof(U1) == sizeof(cfloat) -checkPragmas(U1, pragmas & @["union"]) +checkPragmas(U1, pHeaderBy & @["union"], istype = false) +var u1: U1 +u1.f1 = 5 assert U2 is object assert sizeof(U2) == 256 * sizeof(cint) -checkPragmas(U2, pragmas & @["union"]) +checkPragmas(U2, pHeaderImpBy & @["union"]) +var u2: U2 +u2.f1 = addr a15.a2[0] assert PANEL_WINDOW == 1 assert PANEL_GROUP == 2 From 36faf6226906fccf7232ef7b82102fcd7819b0ad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 30 Mar 2020 15:56:05 -0500 Subject: [PATCH 020/255] Add wrapper test --- tests/tast2.nim | 2 +- tests/wrappers.nims | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/wrappers.nims diff --git a/tests/tast2.nim b/tests/tast2.nim index 4b19429..d77ae39 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -12,7 +12,7 @@ when defined(HEADER): cDefine("HEADER") const flags = " -H -d" - pHeader = @["header:" & path] + pHeader = @["header:" & path.replace("\\", "/")] pHeaderImp = @["importc"] & pHeader else: const diff --git a/tests/wrappers.nims b/tests/wrappers.nims new file mode 100644 index 0000000..37ac686 --- /dev/null +++ b/tests/wrappers.nims @@ -0,0 +1,15 @@ +import os + +let + wrappers = @["genotrance/nimarchive", "genotrance/nimgit2"] + +rmDir("wrappers") +mkDir("wrappers") +withDir("wrappers"): + for wrapper in wrappers: + let + name = wrapper.extractFilename() + exec "git clone https://github.com/" & wrapper + withDir(name): + exec "nimble install -d" + exec "nimble test" \ No newline at end of file From 54854a356adfacdedb85054df7dd5fc3d4eb02da Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 30 Mar 2020 17:14:39 -0500 Subject: [PATCH 021/255] Skip wrapper test on AppVeyor --- nimterop.nimble | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 108d935..8008b27 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -51,6 +51,7 @@ task test, "Test": # getHeader tests withDir("tests"): execCmd("nim e getheader.nims") - execCmd("nim e wrappers.nims") + if not existsEnv("APPVEYOR"): + execCmd("nim e wrappers.nims") docsTask() From ad8557d70e5c8ff3220582b7f78a2822f87d2476 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 1 Apr 2020 21:47:15 -0500 Subject: [PATCH 022/255] ast2 proc pragmas, fix typedef struct X importc, convention selection, ast2 bitfield support --- README.md | 48 ++++++++--------- nimterop.nimble | 12 +++-- nimterop/ast2.nim | 120 ++++++++++++++++++++++++++++++++++++------ nimterop/globals.nim | 2 +- nimterop/toast.nim | 8 ++- tests/include/tast2.h | 10 +++- tests/tast2.nim | 46 +++++++--------- tests/tnimterop_c.nim | 14 ++++- 8 files changed, 183 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index b1fee44..18f74ce 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,6 @@ Check out [template.nim](https://github.com/nimterop/nimterop/blob/master/nimter Refer to the ```tests``` directory for examples on how the library can be used. - The `toast` binary can also be used directly on the CLI: ``` @@ -94,29 +93,30 @@ The `toast` binary can also be used directly on the CLI: Usage: main [optional-params] C/C++ source/header Options: - -h, --help print this cligen-erated help - --help-syntax advanced: prepend,plurals,.. - -k, --check bool false check generated wrapper with compiler - -d, --debug bool false enable debug output - -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" import symbols from library in specified Nim string - -f=, --feature= Features {} flags to enable experimental features - -H, --includeHeader bool false add {.header.} pragma to wrapper - -I=, --includeDirs= strings {} include directory to pass to preprocessor - -m=, --mode= string "cpp" language parser: c or cpp - --nim= string "nim" use a particular Nim executable (default: $PATH/nim) - -c, --nocomments bool false exclude top-level comments from output - -o=, --output= string "" file to output content - default stdout - -a, --past bool false print AST output - -g, --pgrammar bool false print grammar - --pluginSourcePath= string "" nim file to build and load as a plugin - -n, --pnim bool false print Nim output - -E=, --prefix= strings {} strip prefix from identifiers - -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files - -s, --stub bool false stub out undefined type references as objects - -F=, --suffix= strings {} strip suffix from identifiers - -O=, --symOverride= strings {} skip generating specified symbols + -h, --help print this cligen-erated help + --help-syntax advanced: prepend,plurals,.. + -k, --check bool false check generated wrapper with compiler + -C=, --convention= string "cdecl" calling convention for wrapped procs - default: cdecl + -d, --debug bool false enable debug output + -D=, --defines= strings {} definitions to pass to preprocessor + -l=, --dynlib= string "" import symbols from library in specified Nim string + -f=, --feature= Features {} flags to enable experimental features + -H, --includeHeader bool false add {.header.} pragma to wrapper + -I=, --includeDirs= strings {} include directory to pass to preprocessor + -m=, --mode= string "cpp" language parser: c or cpp + --nim= string "nim" use a particular Nim executable - default: $PATH/nim + -c, --nocomments bool false exclude top-level comments from output + -o=, --output= string "" file to output content - default: stdout + -a, --past bool false print AST output + -g, --pgrammar bool false print grammar + --pluginSourcePath= string "" nim file to build and load as a plugin + -n, --pnim bool false print Nim output + -E=, --prefix= strings {} strip prefix from identifiers + -p, --preprocess bool false run preprocessor on header + -r, --recurse bool false process #include files + -s, --stub bool false stub out undefined type references as objects + -F=, --suffix= strings {} strip suffix from identifiers + -O=, --symOverride= strings {} skip generating specified symbols ``` __Implementation Details__ diff --git a/nimterop.nimble b/nimterop.nimble index 8008b27..b20f6a7 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -18,9 +18,9 @@ proc execCmd(cmd: string) = echo "execCmd:" & cmd exec cmd -proc execTest(test: string) = - execCmd "nim c -f -r " & test - execCmd "nim cpp -r " & test +proc execTest(test: string, flags = "") = + execCmd "nim c -f " & flags & " -r " & test + execCmd "nim cpp " & flags & " -r " & test task buildToast, "build toast": execCmd("nim c -f nimterop/toast.nim") @@ -35,11 +35,15 @@ task test, "Test": buildToastTask() execTest "tests/tast2.nim" - execCmd "nim c -f -d:HEADER -r tests/tast2.nim" + execTest "tests/tast2.nim", "-d:HEADER" execTest "tests/tnimterop_c.nim" + execTest "tests/tnimterop_c.nim", "-d:AST2" + execTest "tests/tnimterop_c.nim", "-d:HEADER -d:AST2" + execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" + execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h" execTest "tests/tpcre.nim" # Platform specific tests diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 7eb22a1..4c001f9 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -286,6 +286,24 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", # ), # nkEmpty() # ) + # + # type name* {.bycopy, importc: "abc".} = + # + # nkTypeDef( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkPragma( + # nkIdent("bycopy"), + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("abc") + # ) + # ) + # ) + # ) var pragmas = if nimState.includeHeader: @@ -329,6 +347,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", # nkIdent("*"), # nkIdent(name) # ) + # + # No pragmas here since proc pragmas are elsewhere in the AST result = ident nimState.identifierNodes[name] = result @@ -465,8 +485,17 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) + # Bitfield support - typedef struct { int field: 1; }; + prident = + if node.len > start+1 and node[start+2].getName() == "bitfield_clause": + nimState.newPragmaExpr(node, pident, "bitsize", + newIntNode(nkIntLit, parseInt(nimState.getNodeVal(node[start+2].getAtom())))) + else: + pident + count = node[start+1].getPtrCount() - result.add pident + + result.add prident if count > 0: result.add nimState.newPtrTree(count, tident) else: @@ -527,11 +556,11 @@ proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNo # .. # ) # ), - # nkEmpty() + # nkPragma(...) # ) result = newNode(nkProcTy) result.add nimState.newFormalParams(name, node, rtyp) - result.add newNode(nkEmpty) + result.add nimState.newPragma(node, nimState.gState.convention) proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object @@ -602,6 +631,37 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname # nkEmpty() # ) # ) + # + # type + # X* {.bycopy.} = object + # field1*: cint + # + # nkTypeDef( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkPragma( + # nkIdent("bycopy") + # ) + # ), + # nkEmpty(), + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkRecList( + # nkIdentDefs( + # nkPostfix( + # nkIdent("*"), + # nkIdent("field1") + # ), + # nkIdent("cint"), + # nkEmpty() + # ) + # ) + # ) + # ) let name = typeDef.getIdentName() obj = newNode(nkObjectTy) @@ -893,7 +953,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = # ), # ... # ), - # nkEmpty() + # nkPragma(...) # ) # ) # ) @@ -1069,7 +1129,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # First add struct as object decho("addType(): case 6") - nimState.addTypeObject(node[0], istype = true, union = union) + nimState.addTypeObject(node[0], union = union) if node.len > 1 and nimState.getNodeVal(node[1]) != "": # Add any additional names @@ -1193,6 +1253,7 @@ proc addProc(nimState: NimState, node: TSNode) = let # Only need the ident tree, not nkTypeDef parent name = ident.getIdentName() + origname = nimState.getNodeVal(node[i].getAtom()) # node[i] could have nested pointers tcount = node[i].getPtrCount() @@ -1249,7 +1310,27 @@ proc addProc(nimState: NimState, node: TSNode) = # Proc with return type and params procDef.add nimState.newFormalParams(name, plist, retType) - procDef.add newNode(nkEmpty) # Pragmas + + # Pragmas + let + prident = + if name != origname: + # Explicit {.importc: "origname".} + nimState.newPragma(node[i], "importc", newStrNode(nkStrLit, origname)) + else: + # {.impnameC.} shortcut + nimState.newPragma(node[i], nimState.impShort & "C") + + # Need {.convention.} and {.header.} if applicable + if name != origname: + if nimState.includeHeader(): + # {.impnameHC.} shortcut + nimState.addPragma(node[i], prident, nimState.impShort & "HC") + else: + # {.convention.} + nimState.addPragma(node[i], prident, nimState.gState.convention) + + procDef.add prident procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) @@ -1324,37 +1405,46 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = # Create shortcut pragmas to reduce clutter var hdrPragma: PNode + hdrConvPragma: PNode impPragma = newNode(nkPragma) - impCPragma = newNode(nkPragma) + impConvPragma = newNode(nkPragma) - # {.importc.} + # {.pragma: impname, importc.} nimState.addPragma(root, impPragma, "pragma", nimState.getIdent(nimState.impShort)) nimState.addPragma(root, impPragma, "importc") if nimState.includeHeader(): + # Path to header const nimState.constSection.add nimState.newConstDef( root, fname = nimState.currentHeader, fval = '"' & fullpath & '"') - # {.header: "xxx".} + # {.pragma: impnameH, header: "xxx".} for types when name != origname hdrPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "H")) nimState.addPragma(root, hdrPragma, "header", nimState.getIdent(nimState.currentHeader)) + # Add {.impnameH.} to {.impname.} nimState.addPragma(root, impPragma, nimState.impShort & "H") - # {.importc.} + {.cdecl.} for procs - nimState.addPragma(root, impCPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) - nimState.addPragma(root, impCPragma, nimState.impShort) - nimState.addPragma(root, impCPragma, "cdecl") + # {.pragma: impnameHC, impnameH, convention.} for procs when name != origname + hdrConvPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "HC")) + nimState.addPragma(root, hdrConvPragma, nimState.impShort & "H") + nimState.addPragma(root, hdrConvPragma, nimState.gState.convention) + + # {.pragma: impnameC, impname, convention.} for procs + nimState.addPragma(root, impConvPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) + nimState.addPragma(root, impConvPragma, nimState.impShort) + nimState.addPragma(root, impConvPragma, nimState.gState.convention) if nimState.gState.dynlib.nBl: # {.dynlib.} for DLLs - nimState.addPragma(root, impCPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + nimState.addPragma(root, impConvPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib)) # Add all pragma shortcuts to output if not hdrPragma.isNil: nimState.pragmaSection.add hdrPragma + nimState.pragmaSection.add hdrConvPragma nimState.pragmaSection.add impPragma - nimState.pragmaSection.add impCPragma + nimState.pragmaSection.add impConvPragma proc printNimHeader*(gState: State) = # Top level output with context info diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 68f18b0..0e4827f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -59,7 +59,7 @@ type debug*, includeHeader*, nocache*, nocomments*, past*, preprocess*, pnim*, recurse*: bool - code*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string + code*, convention*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string feature*: seq[Feature] diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 323b4ec..a82a4ea 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -54,6 +54,7 @@ proc process(gState: State, path: string, astTable: AstTable) = # CLI processing with default values proc main( check = false, + convention = "cdecl", debug = false, defines: seq[string] = @[], dynlib: string = "", @@ -79,6 +80,7 @@ proc main( # Setup global state with arguments var gState = State( + convention: convention, debug: debug, defines: defines, dynlib: dynlib, @@ -189,6 +191,7 @@ when isMainModule: import cligen dispatch(main, help = { "check": "check generated wrapper with compiler", + "convention": "calling convention for wrapped procs - default: cdecl", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", "dynlib": "import symbols from library in specified Nim string", @@ -196,9 +199,9 @@ when isMainModule: "includeHeader": "add {.header.} pragma to wrapper", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", - "nim": "use a particular Nim executable (default: $PATH/nim)", + "nim": "use a particular Nim executable - default: $PATH/nim", "nocomments": "exclude top-level comments from output", - "output": "file to output content - default stdout", + "output": "file to output content - default: stdout", "past": "print AST output", "pgrammar": "print grammar", "pluginSourcePath": "nim file to build and load as a plugin", @@ -212,6 +215,7 @@ when isMainModule: "symOverride": "skip generating specified symbols" }, short = { "check": 'k', + "convention": 'C', "debug": 'd', "defines": 'D', "dynlib": 'l', diff --git a/tests/include/tast2.h b/tests/include/tast2.h index db539c7..1518a85 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -1,3 +1,7 @@ +#ifdef __cplusplus +extern "C" { +#endif + #define A 1 #define B 1.0 #define C 0x10 @@ -26,7 +30,7 @@ struct A4 { typedef char *A9p[3]; //, A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; -typedef struct A1 *A111[12]; +typedef struct A0 *A111[12]; typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); typedef int A13(int, int, void (*func)(void)); @@ -212,4 +216,8 @@ typedef enum VSPresetFormat { //struct A2 test_proc1(struct A0 a); +#endif + +#ifdef __cplusplus +} #endif \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index d77ae39..d98907f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -11,7 +11,7 @@ const when defined(HEADER): cDefine("HEADER") const - flags = " -H -d" + flags = " -H" pHeader = @["header:" & path.replace("\\", "/")] pHeaderImp = @["importc"] & pHeader else: @@ -183,21 +183,15 @@ a11 = addr a9p assert A111 is array[12, ptr A1] checkPragmas(A111, pHeaderImp) var a111: A111 -a111[11] = addr a1 +a111[11] = addr a0 -assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint -checkPragmas(A12, pHeaderImp) -when not defined(HEADER): - # Unclear why this fails - # request for member ‘ClE_0’ in something not a structure or union - var a12: A12 +assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint {.cdecl.}): ptr ptr cint {.cdecl.} +checkPragmas(A12, pHeaderImp & "cdecl") +var a12: A12 -assert A13 is proc(a1: cint, a2: cint, `func`: proc()): cint -checkPragmas(A13, pHeaderImp) -when not defined(HEADER): - # Unclear why this fails - # request for member ‘ClE_0’ in something not a structure or union - var a13: A13 +assert A13 is proc(a1: cint, a2: cint, `func`: proc() {.cdecl.}): cint {.cdecl.} +checkPragmas(A13, pHeaderImp & "cdecl") +var a13: A13 assert A14 is object testFields(A14, "a1:cchar") @@ -216,20 +210,16 @@ a15.a2[0] = addr a15i assert A16 is object testFields(A16, "f1:cchar") -checkPragmas(A16, pHeaderImpBy) -when not defined(HEADER): - # Similar to A2 - var a16: A16 - a16.f1 = 's' +checkPragmas(A16, pHeaderBy, istype = false) +var a16: A16 +a16.f1 = 's' assert A17 is object testFields(A17, "a1|a2:cstring|array[1, ptr cint]") -checkPragmas(A17, pHeaderImpBy) -when not defined(HEADER): - # Similar to A2 - var a17: A17 - a17.a1 = "hello".cstring - a17.a2[0] = addr a15i +checkPragmas(A17, pHeaderBy, istype = false) +var a17: A17 +a17.a1 = "hello".cstring +a17.a2[0] = addr a15i assert A18 is A17 checkPragmas(A18, pHeaderImp) @@ -254,7 +244,7 @@ a19p = addr a19 assert A20 is object testFields(A20, "a1:cchar") -checkPragmas(A20, pHeaderImpBy) +checkPragmas(A20, pHeaderBy, istype = false) var a20: A20 a20.a1 = 'a' @@ -270,7 +260,7 @@ a21p = addr a20 assert A22 is object testFields(A22, "f1|f2:ptr ptr cint|array[123 + 132, ptr cint]") -checkPragmas(A22, pHeaderImpBy) +checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] @@ -282,7 +272,7 @@ u1.f1 = 5 assert U2 is object assert sizeof(U2) == 256 * sizeof(cint) -checkPragmas(U2, pHeaderImpBy & @["union"]) +checkPragmas(U2, pHeaderBy & @["union"], istype = false) var u2: U2 u2.f1 = addr a15.a2[0] diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index 732efbf..afd73e2 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -41,7 +41,17 @@ cOverride: proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.} proc weirdfunc2(mango: ptr ptr cchar): int {.importc.} -cImport(cSearchPath("test.h")) +# includeHeader +const header = + when defined(HEADER): " -H" + else: "" + +# Test AST2 +const mode = + when defined(AST2): " -f:ast2" + else: "" + +cImport(cSearchPath("test.h"), flags = header & mode) check TEST_INT == 512 check TEST_FLOAT == 5.12 @@ -65,7 +75,7 @@ var ct: CUSTTYPE cct: CCUSTTYPE - s0: STRUCT0 + s0: ptr STRUCT0 s1: STRUCT1 s2: STRUCT2 s3: STRUCT3 From 5c20ce7b60225bcc5638c78a7632833f11b2aa83 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Apr 2020 18:17:36 -0500 Subject: [PATCH 023/255] ast2 test fix for osx, catch parseString errors, handle type field comments --- nimterop.nimble | 20 +++++++++++------- nimterop/ast2.nim | 47 +++++++++++++++++++++++++++++-------------- tests/include/tast2.h | 4 ++-- tests/tnimterop_c.nim | 13 ++---------- tests/tpcre.nim | 3 ++- 5 files changed, 51 insertions(+), 36 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index b20f6a7..be76135 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -19,14 +19,17 @@ proc execCmd(cmd: string) = exec cmd proc execTest(test: string, flags = "") = - execCmd "nim c -f " & flags & " -r " & test - execCmd "nim cpp " & flags & " -r " & test + execCmd "nim c --hints:off -f " & flags & " -r " & test + execCmd "nim cpp --hints:off " & flags & " -r " & test task buildToast, "build toast": - execCmd("nim c -f nimterop/toast.nim") + execCmd("nim c --hints:off -f nimterop/toast.nim") task bt, "build toast": - execCmd("nim c -d:danger nimterop/toast.nim") + execCmd("nim c --hints:off -d:danger nimterop/toast.nim") + +task btd, "build toast": + execCmd("nim c --hints:off nimterop/toast.nim") task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") @@ -38,13 +41,16 @@ task test, "Test": execTest "tests/tast2.nim", "-d:HEADER" execTest "tests/tnimterop_c.nim" - execTest "tests/tnimterop_c.nim", "-d:AST2" - execTest "tests/tnimterop_c.nim", "-d:HEADER -d:AST2" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\"" - execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" + execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h" + execTest "tests/tpcre.nim" + #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" + #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2 -H\"" # Platform specific tests when defined(Windows): diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4c001f9..0509c7c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -2,7 +2,7 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, modulegraphs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, options, parser, renderer] import "."/treesitter/api @@ -17,9 +17,19 @@ proc getPtrType*(str: string): string = else: str +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting + raise newException(Exception, "") + proc parseString(nimState: NimState, str: string): PNode = - # Parse a string into Nim AST - result = parseString(str, nimState.identCache, nimState.config) + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + discard proc getLit*(nimState: NimState, str: string): PNode = # Used to convert #define literals into const and expressions @@ -33,9 +43,8 @@ proc getLit*(nimState: NimState, str: string): PNode = elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float result = newFloatNode(nkFloatLit, parseFloat(str)) - # # TODO - hex becomes int on render - # elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - # result = newIntNode(nkIntLit, parseHexInt(str)) + elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + result = nimState.parseString(str) elif str.contains(re"^'[[:ascii:]]'$"): # char result = newNode(nkCharLit) @@ -46,8 +55,9 @@ proc getLit*(nimState: NimState, str: string): PNode = else: result = nimState.parseString(nimState.getNimExpression(str)) - if result.isNil: - result = newNode(nkNilLit) + + if result.isNil: + result = newNode(nkNilLit) proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -132,7 +142,9 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind != nkNilLit: + elif valident.kind in {nkCharLit .. nkStrLit} or + (valident.kind == nkStmtList and valident.len > 0 and + valident[0].kind in {nkCharLit .. nkStrLit}): if nimState.addNewIdentifer(name): # const X* = Y # @@ -147,7 +159,11 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode result = newNode(nkConstDef) result.add ident result.add newNode(nkEmpty) - result.add valident + if valident.kind == nkStmtList and valident.len == 1: + # Collapse single line statement + result.add valident[0] + else: + result.add valident else: necho &"# const '{origname}' is duplicate, skipped" else: @@ -576,11 +592,12 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = result = newNode(nkRecList) for i in 0 ..< node.len: - # Add nkIdentDefs for each field - let - field = nimState.newIdentDefs(name, node[i], i, exported = true) - if not field.isNil: - result.add field + if node[i].getName() == "field_declaration": + # Add nkIdentDefs for each field + let + field = nimState.newIdentDefs(name, node[i], i, exported = true) + if not field.isNil: + result.add field proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 1518a85..e1d7b4e 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -157,11 +157,11 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p; typedef struct A20 { char a1; } A20, A21, *A21p; //Expression -typedef struct A22 { int **f1; int *f2[123+132]; } A22; +typedef struct A22 { const int **f1; int *f2[123+132]; } A22; //Unions union U1 {int f1; float f2; }; -typedef union U2 { int **f1; int abc[123+132]; } U2; +typedef union U2 { const int **f1; int abc[123+132]; } U2; // Enums diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index afd73e2..fe4e9ee 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -41,17 +41,8 @@ cOverride: proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.} proc weirdfunc2(mango: ptr ptr cchar): int {.importc.} -# includeHeader -const header = - when defined(HEADER): " -H" - else: "" - -# Test AST2 -const mode = - when defined(AST2): " -f:ast2" - else: "" - -cImport(cSearchPath("test.h"), flags = header & mode) +const FLAGS {.strdefine.} = "" +cImport(cSearchPath("test.h"), flags = FLAGS) check TEST_INT == 512 check TEST_FLOAT == 5.12 diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 0aa5a3b..21f3388 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -30,7 +30,8 @@ cPlugin: proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = sym.name = sym.name.replace("pcre_", "") -cImport(pcreH, dynlib="dynpcre") +const FLAGS {.strdefine.} = "" +cImport(pcreH, dynlib="dynpcre", flags = FLAGS) echo version() From ced9c148287a50356eb5ace81ef6582cc0660163 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Apr 2020 18:17:36 -0500 Subject: [PATCH 024/255] ast2 test fix for osx, catch parseString errors, handle type field comments --- nimterop.nimble | 20 +++++++++++------- nimterop/ast2.nim | 47 +++++++++++++++++++++++++++++-------------- tests/include/tast2.h | 10 ++++----- tests/tnimterop_c.nim | 13 ++---------- tests/tpcre.nim | 3 ++- 5 files changed, 54 insertions(+), 39 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index b20f6a7..be76135 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -19,14 +19,17 @@ proc execCmd(cmd: string) = exec cmd proc execTest(test: string, flags = "") = - execCmd "nim c -f " & flags & " -r " & test - execCmd "nim cpp " & flags & " -r " & test + execCmd "nim c --hints:off -f " & flags & " -r " & test + execCmd "nim cpp --hints:off " & flags & " -r " & test task buildToast, "build toast": - execCmd("nim c -f nimterop/toast.nim") + execCmd("nim c --hints:off -f nimterop/toast.nim") task bt, "build toast": - execCmd("nim c -d:danger nimterop/toast.nim") + execCmd("nim c --hints:off -d:danger nimterop/toast.nim") + +task btd, "build toast": + execCmd("nim c --hints:off nimterop/toast.nim") task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") @@ -38,13 +41,16 @@ task test, "Test": execTest "tests/tast2.nim", "-d:HEADER" execTest "tests/tnimterop_c.nim" - execTest "tests/tnimterop_c.nim", "-d:AST2" - execTest "tests/tnimterop_c.nim", "-d:HEADER -d:AST2" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\"" - execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" + execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h" + execTest "tests/tpcre.nim" + #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" + #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2 -H\"" # Platform specific tests when defined(Windows): diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4c001f9..0509c7c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -2,7 +2,7 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, modulegraphs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, options, parser, renderer] import "."/treesitter/api @@ -17,9 +17,19 @@ proc getPtrType*(str: string): string = else: str +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting + raise newException(Exception, "") + proc parseString(nimState: NimState, str: string): PNode = - # Parse a string into Nim AST - result = parseString(str, nimState.identCache, nimState.config) + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + discard proc getLit*(nimState: NimState, str: string): PNode = # Used to convert #define literals into const and expressions @@ -33,9 +43,8 @@ proc getLit*(nimState: NimState, str: string): PNode = elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float result = newFloatNode(nkFloatLit, parseFloat(str)) - # # TODO - hex becomes int on render - # elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - # result = newIntNode(nkIntLit, parseHexInt(str)) + elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + result = nimState.parseString(str) elif str.contains(re"^'[[:ascii:]]'$"): # char result = newNode(nkCharLit) @@ -46,8 +55,9 @@ proc getLit*(nimState: NimState, str: string): PNode = else: result = nimState.parseString(nimState.getNimExpression(str)) - if result.isNil: - result = newNode(nkNilLit) + + if result.isNil: + result = newNode(nkNilLit) proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -132,7 +142,9 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind != nkNilLit: + elif valident.kind in {nkCharLit .. nkStrLit} or + (valident.kind == nkStmtList and valident.len > 0 and + valident[0].kind in {nkCharLit .. nkStrLit}): if nimState.addNewIdentifer(name): # const X* = Y # @@ -147,7 +159,11 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode result = newNode(nkConstDef) result.add ident result.add newNode(nkEmpty) - result.add valident + if valident.kind == nkStmtList and valident.len == 1: + # Collapse single line statement + result.add valident[0] + else: + result.add valident else: necho &"# const '{origname}' is duplicate, skipped" else: @@ -576,11 +592,12 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = result = newNode(nkRecList) for i in 0 ..< node.len: - # Add nkIdentDefs for each field - let - field = nimState.newIdentDefs(name, node[i], i, exported = true) - if not field.isNil: - result.add field + if node[i].getName() == "field_declaration": + # Add nkIdentDefs for each field + let + field = nimState.newIdentDefs(name, node[i], i, exported = true) + if not field.isNil: + result.add field proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 1518a85..84a4604 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -45,11 +45,11 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p; typedef struct A20 { char a1; } A20, A21, *A21p; //Expression -typedef struct A22 { int **f1; int *f2[123+132]; } A22; +typedef struct A22 { const int **f1; int *f2[123+132]; } A22; //Unions union U1 {int f1; float f2; }; -typedef union U2 { int **f1; int abc[123+132]; } U2; +typedef union U2 { const int **f1; int abc[123+132]; } U2; // Enums @@ -157,11 +157,11 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p; typedef struct A20 { char a1; } A20, A21, *A21p; //Expression -typedef struct A22 { int **f1; int *f2[123+132]; } A22; +typedef struct A22 { const int **f1; int *f2[123+132]; } A22; //Unions union U1 {int f1; float f2; }; -typedef union U2 { int **f1; int abc[123+132]; } U2; +typedef union U2 { const int **f1; int abc[123+132]; } U2; // Enums @@ -220,4 +220,4 @@ typedef enum VSPresetFormat { #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index afd73e2..fe4e9ee 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -41,17 +41,8 @@ cOverride: proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.} proc weirdfunc2(mango: ptr ptr cchar): int {.importc.} -# includeHeader -const header = - when defined(HEADER): " -H" - else: "" - -# Test AST2 -const mode = - when defined(AST2): " -f:ast2" - else: "" - -cImport(cSearchPath("test.h"), flags = header & mode) +const FLAGS {.strdefine.} = "" +cImport(cSearchPath("test.h"), flags = FLAGS) check TEST_INT == 512 check TEST_FLOAT == 5.12 diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 0aa5a3b..21f3388 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -30,7 +30,8 @@ cPlugin: proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = sym.name = sym.name.replace("pcre_", "") -cImport(pcreH, dynlib="dynpcre") +const FLAGS {.strdefine.} = "" +cImport(pcreH, dynlib="dynpcre", flags = FLAGS) echo version() From 534acf9259a233eb5cf0e59ae754b4c036893f81 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Apr 2020 19:26:38 -0500 Subject: [PATCH 025/255] ast2 var of proc type, multiple proc type block --- nimterop/ast2.nim | 423 +++++++++++++++++++++++++++--------------- nimterop/globals.nim | 2 +- tests/include/tast2.h | 12 +- tests/tast2.nim | 19 +- tests/tpcre.nim | 8 +- tests/tsoloud.nim | 4 +- 6 files changed, 310 insertions(+), 158 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 0509c7c..2182fec 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -264,11 +264,13 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: seq[ result.add ident result.add nimState.newPragma(node, pragmas) -proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[], istype = false): PNode = - # Create nkTypeDef PNode with first ident if `nskType` else just create an nkPostfix node for `nskProc` +proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[], istype = false): PNode = + # Create nkTypeDef PNode with first ident if `nskType` + # Create nkIdentDefs PNode with first ident if `nskVar` + # Create an nkPostfix node for `nskProc` # # If `fname`, use it instead of node.getAtom() for name - # If `pragmas`, add as nkPragmaExpr but only if `nskType` since procs add pragmas elsewhere + # If `pragmas`, add as nkPragmaExpr but not for `nskProc` since procs add pragmas elsewhere # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} when includeHeader let (tname, torigname, info) = nimState.getNameInfo(node.getAtom(), kind) @@ -318,7 +320,8 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", # nkStrLit("abc") # ) # ) - # ) + # ), + # nkEmpty() # ) var pragmas = @@ -356,6 +359,42 @@ proc newTypeIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", result = newNode(nkTypeDef) result.add prident result.add newNode(nkEmpty) + elif kind == nskVar: + # var name* {.importc: "abc".} + # + # nkIdentDefs( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("abc") + # ) + # ) + # ) + # ) + let + prident = block: + var + prident: PNode + if name != origname: + # Add importc pragma since name changed + prident = nimState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) + if nimState.includeHeader(): + # Add header + nimState.addPragma(node, prident[1], nimState.impShort & "H") + else: + # Only need impShort since no name change + prident = nimState.newPragmaExpr(node, ident, nimState.impShort) + if pragmas.nBl: + nimState.addPragma(node, prident[1], pragmas) + prident + + result = newNode(nkIdentDefs) + result.add prident elif kind == nskProc: # name* # @@ -427,7 +466,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add typ proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode -proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode +proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field @@ -523,7 +562,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeProc(name, node) + result.add nimState.getTypeProc(name, node[1], rnode = node[0]) result.add newNode(nkEmpty) elif not adecl.isNil: # Named param, array type @@ -629,7 +668,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname typeDef = if typeDef.isNil: - nimState.newTypeIdent(node, fname = fname, pragmas = pragmas, istype = istype) + nimState.newXIdent(node, fname = fname, pragmas = pragmas, istype = istype) else: typeDef @@ -700,7 +739,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) typedef[0] = npexpr else: - # includeHeader already added impShort in newTypeIdent() + # includeHeader already added impShort in newXIdent() nimState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add @@ -750,7 +789,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specific type let # node[i] = identifer = name - typeDef = nimState.newTypeIdent(node[i], istype = true) + typeDef = nimState.newXIdent(node[i], istype = true) if not typeDef.isNil: let @@ -856,7 +895,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = decho("addTypeArray()") let # node[1] = identifer = name - typeDef = nimState.newTypeIdent(node[1], istype = true) + typeDef = nimState.newXIdent(node[1], istype = true) if not typeDef.isNil: let @@ -889,17 +928,19 @@ proc addTypeArray(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = +proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # Create proc type tree + # + # `rnode` is the return type let - # node[0] = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType, parent = name) + # rnode = identifier = return type name + (rname, _, rinfo) = nimState.getNameInfo(rnode.getAtom(), nskType, parent = name) # Parameter list - plist = node[1].anyChildInTree("parameter_list") + plist = node.anyChildInTree("parameter_list") # node[1] could have nested pointers - tcount = node[1].getPtrCount() + tcount = node.getPtrCount() # Name could be nested pointer to function # @@ -914,7 +955,7 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # ) # ) # ) - ncount = node[1].getAtom().tsNodeParent().getPtrCount(reverse = true) + ncount = node.getAtom().tsNodeParent().getPtrCount(reverse = true) # Return type var @@ -936,49 +977,57 @@ proc addTypeProc(nimState: NimState, node: TSNode) = # Add a type of proc type decho("addTypeProc()") let - # node[1] = identifier = name - typeDef = nimState.newTypeIdent(node[1], istype = true) + start = getStartAtom(node) - if not typeDef.isNil: + # node[start] = return type + rnode = node[start] + + # Could have multiple types, comma separated + for i in start+1 ..< node.len: let - name = typeDef.getIdentName() + # node[i] = identifier = name + typeDef = nimState.newXIdent(node[i], istype = true) - procTy = nimState.getTypeProc(name, node) + if not typeDef.isNil: + let + name = typeDef.getIdentName() - typeDef.add procTy + procTy = nimState.getTypeProc(name, node[i], rnode) - # type X* = proc(a1: Y, a2: Z): P - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkProcTy( - # nkFormalParams( - # nkPtrTy( # optional, nested - # nkIdent(retType) - # ), - # nkIdentDefs( - # nkIdent(param), - # nkPtrTy( - # nkIdent(ptype) - # ), - # nkEmpty() - # ), - # ... - # ), - # nkPragma(...) - # ) - # ) - # ) + typeDef.add procTy - # nkTypeSection.add - nimState.typeSection.add typeDef + # type X* = proc(a1: Y, a2: Z): P + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkProcTy( + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkPragma(...) + # ) + # ) + # ) - nimState.printDebug(typeDef) + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode, union = false) = decho("addType()") @@ -1253,108 +1302,184 @@ proc addEnum(nimState: NimState, node: TSNode) = if node.getName() == "type_definition" and node.len > 1: nimState.addTypeTyped(node, ftname = name, offset = offset) -proc addProc(nimState: NimState, node: TSNode) = +proc addProcVar(nimState: NimState, node, rnode: TSNode) = + # Add a proc variable + decho("addProcVar()") + let + # node = identifier = name + identDefs = nimState.newXIdent(node, kind = nskVar, istype = true) + + if not identDefs.isNil: + let + name = identDefs.getIdentName() + # origname = nimState.getNodeVal(node.getAtom()) + + procTy = nimState.getTypeProc(name, node, rnode) + + identDefs.add procTy + identDefs.add newNode(nkEmpty) + + # var X* {.importc: "_X": proc(a1: Y, a2: Z): P {.cdecl.} + # + # nkIdentDefs( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("_X") + # ) + # ) + # ), + # nkProcTy( + # nkFormalParams( + # nkIdent("P"), + # nkIdentDefs( + # nkIdent("a1"), + # nkIdent("Y"), + # nkEmpty() + # ), + # nkIdentDefs( + # nkIdent("a2"), + # nkIdent("Z"), + # nkEmpty() + # ) + # ), + # nkPragma( + # nkIdent("cdecl") + # ) + # ), + # nkEmpty() + # ) + + # nkVarSection.add + nimState.varSection.add identDefs + + nimState.printDebug(identDefs) + +proc addProc(nimState: NimState, node, rnode: TSNode) = # Add a proc + # + # `node` is the `nth` child of (declaration) + # `rnode` is the return value node, the first child of (declaration) decho("addProc()") + let + # node = identifier = name + ident = nimState.newXIdent(node, kind = nskProc) + + if not ident.isNil: + let + # Only need the ident tree, not nkTypeDef parent + name = ident.getIdentName() + origname = nimState.getNodeVal(node.getAtom()) + + # node could have nested pointers + tcount = node.getPtrCount() + + # rnode = identifier = return type name + (rname, _, rinfo) = nimState.getNameInfo(rnode.getAtom(), nskType, parent = name) + + # Parameter list + plist = node.anyChildInTree("parameter_list") + + procDef = newNode(nkProcDef) + + # proc X(a1: Y, a2: Z): P {.pragma.} + # + # nkProcDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkEmpty(), + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkPragma(...), + # nkEmpty(), + # nkEmpty() + # ) + + procDef.add ident + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) + + # Return type + var + retType = + if rname == "object" and tcount == 0: + # void func(..) + newNode(nkEmpty) + else: + nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = nimState.newPtrTree(tcount, retType) + + # Proc with return type and params + procDef.add nimState.newFormalParams(name, plist, retType) + + # Pragmas + let + prident = + if name != origname: + # Explicit {.importc: "origname".} + nimState.newPragma(node, "importc", newStrNode(nkStrLit, origname)) + else: + # {.impnameC.} shortcut + nimState.newPragma(node, nimState.impShort & "C") + + # Need {.convention.} and {.header.} if applicable + if name != origname: + if nimState.includeHeader(): + # {.impnameHC.} shortcut + nimState.addPragma(node, prident, nimState.impShort & "HC") + else: + # {.convention.} + nimState.addPragma(node, prident, nimState.gState.convention) + + procDef.add prident + procDef.add newNode(nkEmpty) + procDef.add newNode(nkEmpty) + + # nkProcSection.add + nimState.procSection.add procDef + + nimState.printDebug(procDef) + +proc addDecl(nimState: NimState, node: TSNode) = + # Add a declaration + decho("addDecl()") nimState.printDebug(node) let start = getStartAtom(node) for i in start+1 ..< node.len: - let - # node[i] = identifier = name - ident = nimState.newTypeIdent(node[i], kind = nskProc) - - if not ident.isNil: - let - # Only need the ident tree, not nkTypeDef parent - name = ident.getIdentName() - origname = nimState.getNodeVal(node[i].getAtom()) - - # node[i] could have nested pointers - tcount = node[i].getPtrCount() - - # node[start] = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) - - # Parameter list - plist = node[i].anyChildInTree("parameter_list") - - procDef = newNode(nkProcDef) - - # proc X(a1: Y, a2: Z): P {.pragma.} - # - # nkProcDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkEmpty(), - # nkFormalParams( - # nkPtrTy( # optional, nested - # nkIdent(retType) - # ), - # nkIdentDefs( - # nkIdent(param), - # nkPtrTy( - # nkIdent(ptype) - # ), - # nkEmpty() - # ), - # ... - # ), - # nkPragma(...), - # nkEmpty(), - # nkEmpty() - # ) - - procDef.add ident - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) - - # Return type - var - retType = - if rname == "object" and tcount == 0: - # void func(..) - newNode(nkEmpty) - else: - nimState.getIdent(rname, rinfo, exported = false) - if tcount > 0: - retType = nimState.newPtrTree(tcount, retType) - - # Proc with return type and params - procDef.add nimState.newFormalParams(name, plist, retType) - - # Pragmas - let - prident = - if name != origname: - # Explicit {.importc: "origname".} - nimState.newPragma(node[i], "importc", newStrNode(nkStrLit, origname)) - else: - # {.impnameC.} shortcut - nimState.newPragma(node[i], nimState.impShort & "C") - - # Need {.convention.} and {.header.} if applicable - if name != origname: - if nimState.includeHeader(): - # {.impnameHC.} shortcut - nimState.addPragma(node[i], prident, nimState.impShort & "HC") - else: - # {.convention.} - nimState.addPragma(node[i], prident, nimState.gState.convention) - - procDef.add prident - procDef.add newNode(nkEmpty) - procDef.add newNode(nkEmpty) - - # nkProcSection.add - nimState.procSection.add procDef - - nimState.printDebug(procDef) + if not node[i].firstChildInTree("function_declarator").isNil: + # Proc declaration - var or actual proc + if node[i].getAtom().getPxName(1) == "pointer_declarator": + # proc var + nimState.addProcVar(node[i], node[start]) + else: + # proc + nimState.addProc(node[i], node[start]) + else: + # Regular var + discard proc processNode(nimState: NimState, node: TSNode): bool = result = true @@ -1376,7 +1501,7 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "enum_specifier": nimState.addEnum(node) of "declaration": - nimState.addProc(node) + nimState.addDecl(node) else: # Unknown result = false @@ -1496,11 +1621,12 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = nimstate.graph = newModuleGraph(nimState.identCache, nimState.config) # Initialize all section PNodes - nimState.pragmaSection = newNode(nkStmtList) nimState.constSection = newNode(nkConstSection) nimState.enumSection = newNode(nkStmtList) + nimState.pragmaSection = newNode(nkStmtList) nimState.procSection = newNode(nkStmtList) nimState.typeSection = newNode(nkTypeSection) + nimState.varSection = newNode(nkVarSection) # Setup pragmas nimState.setupPragmas(root, fp) @@ -1518,6 +1644,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = tree.add nimState.constSection tree.add nimState.pragmaSection tree.add nimState.typeSection + tree.add nimState.varSection tree.add nimState.procSection gecho tree.renderTree() diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 0e4827f..701db61 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -82,7 +82,7 @@ type # Nim compiler objects when not declared(CIMPORT): - pragmaSection*, constSection*, enumSection*, procSection*, typeSection*: PNode + constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode identCache*: IdentCache config*: ConfigRef graph*: ModuleGraph diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 84a4604..3f46849 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -32,7 +32,10 @@ typedef char *A10[3][6]; typedef char *(*A11)[3]; typedef struct A0 *A111[12]; -typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int + **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), + **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); typedef int A13(int, int, void (*func)(void)); struct A14 { volatile char a1; }; @@ -104,8 +107,11 @@ typedef enum VSPresetFormat { //struct A2 test_proc1(struct A0 a); - - +// Proc vars +void + *(*pcre_malloc)(size_t), + (*pcre_free)(void *), + *(*pcre_stack_malloc)(size_t); diff --git a/tests/tast2.nim b/tests/tast2.nim index d98907f..28ea7c5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -189,6 +189,14 @@ assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, checkPragmas(A12, pHeaderImp & "cdecl") var a12: A12 +assert A121 is proc(a1: cfloat, b: cfloat, c: ptr cfloat, a4: ptr cfloat, count: array[4, ptr cfloat], `func`: proc(a1: cfloat, a2: cfloat): cfloat {.cdecl.}): ptr ptr cint {.cdecl.} +checkPragmas(A121, pHeaderImp & "cdecl") +var a121: A121 + +assert A122 is proc(a1: cchar, b: cchar, c: cstring, a4: cstring, count: array[4, cstring], `func`: proc(a1: cchar, a2: cchar): cchar {.cdecl.}): ptr ptr cint {.cdecl.} +checkPragmas(A122, pHeaderImp & "cdecl") +var a122: A122 + assert A13 is proc(a1: cint, a2: cint, `func`: proc() {.cdecl.}): cint {.cdecl.} checkPragmas(A13, pHeaderImp & "cdecl") var a13: A13 @@ -291,4 +299,13 @@ assert cmGray == 1000000 assert pfGray16 == 1000011 assert pfYUV422P8 == pfYUV420P8 + 1 assert pfRGB27 == cmRGB.VSPresetFormat + 11 -assert pfCompatYUY2 == pfCompatBGR32 + 1 \ No newline at end of file +assert pfCompatYUY2 == pfCompatBGR32 + 1 + +assert pcre_malloc is proc(a1: uint): pointer {.cdecl.} +checkPragmas(pcre_malloc, @["importc", "cdecl"] & pHeader) + +assert pcre_free is proc(a1: pointer) {.cdecl.} +checkPragmas(pcre_free, @["importc", "cdecl"] & pHeader) + +assert pcre_stack_malloc is proc(a1: uint): pointer {.cdecl.} +checkPragmas(pcre_stack_malloc, @["importc", "cdecl"] & pHeader) \ No newline at end of file diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 21f3388..f896e9c 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -35,7 +35,9 @@ cImport(pcreH, dynlib="dynpcre", flags = FLAGS) echo version() -proc my_malloc(a1: uint) {.cdecl.} = - discard +when FLAGS.len != 0: + # Legacy algorithm is broken - does not convert void * return to pointer + proc my_malloc(a1: uint): pointer {.cdecl.} = + discard -malloc = my_malloc + malloc = my_malloc diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index 90cf82d..9823531 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -1,4 +1,4 @@ -import os, nimterop/[cimport, build, paths] +import os, nimterop/[cimport, build] const baseDir = getProjectCacheDir("nimterop" / "tests" / "soloud") @@ -6,7 +6,7 @@ const src = baseDir/"src" static: - gitPull("https://github.com/jarikomppa/soloud", baseDir, "include/*\nsrc/*\n") + gitPull("https://github.com/jarikomppa/soloud", baseDir, "include/*\nsrc/*\n", checkout = "RELEASE_20200207") cDebug() cDisableCaching() From e911a9ee92a92beeb66bc716949c4f417fb0f816 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Apr 2020 23:55:44 -0500 Subject: [PATCH 026/255] ast2 multiple array type block --- nimterop/ast2.nim | 86 +++++++++++++++++++++++-------------------- tests/include/tast2.h | 37 +++++++++++++++++-- tests/tast2.nim | 26 ++++++++++--- 3 files changed, 100 insertions(+), 49 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2182fec..7becd04 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -465,7 +465,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add size result.add typ -proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode +proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = @@ -562,7 +562,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeProc(name, node[1], rnode = node[0]) + result.add nimState.getTypeProc(name, node[start+1], node[start]) result.add newNode(nkEmpty) elif not adecl.isNil: # Named param, array type @@ -570,7 +570,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeArray(node, name) + result.add nimState.getTypeArray(node[start+1], node[start], name) result.add newNode(nkEmpty) else: result = nil @@ -838,24 +838,22 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = else: nimState.addTypeObject(node, typeDef = typeDef, istype = true) -proc getTypeArray(nimState: NimState, node: TSNode, name: string): PNode = +proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode = # Create array type tree let - start = getStartAtom(node) - - # node[start] = identifier = type name - (tname, _, info) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + # tnode = identifier = type name + (tname, _, info) = nimState.getNameInfo(tnode.getAtom(), nskType, parent = name) ident = nimState.getIdent(tname, info, exported = false) # Top-most array declarator - adecl = node[start+1].firstChildInTree("array_declarator") + adecl = node.firstChildInTree("array_declarator") - # node[start+1] could have nested arrays + # node could have nested arrays acount = adecl.getArrayCount() innermost = adecl.mostNestedChildInTree() - # node[start+1] could have nested pointers - type - tcount = node[start+1].getPtrCount() + # node could have nested pointers - type + tcount = node.getPtrCount() # Name could be nested pointer to array # @@ -894,39 +892,47 @@ proc addTypeArray(nimState: NimState, node: TSNode) = # Add a type of array type decho("addTypeArray()") let - # node[1] = identifer = name - typeDef = nimState.newXIdent(node[1], istype = true) + start = getStartAtom(node) - if not typeDef.isNil: + # node[start] = type name + tnode = node[start] + + # Could have multiple types, comma separated + for i in start+1 ..< node.len: let - name = typeDef.getIdentName() - typ = nimState.getTypeArray(node, name) + # node[i] = identifer = name + typeDef = nimState.newXIdent(node[i], istype = true) - typeDef.add typ + if not typeDef.isNil: + let + name = typeDef.getIdentName() + typ = nimState.getTypeArray(node[i], tnode, name) - # type X* = [ptr] array[x, [ptr] Y] - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkBracketExpr( - # nkIdent("array") - # nkXLit(x), - # nkPtrTy( # optional, nested - # nkIdent("Y") - # ) - # ) - # ) - # ) + typeDef.add typ - # nkTypeSection.add - nimState.typeSection.add typeDef + # type X* = [ptr] array[x, [ptr] Y] + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkBracketExpr( + # nkIdent("array") + # nkXLit(x), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # ) + # ) - nimState.printDebug(typeDef) + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # Create proc type tree @@ -939,7 +945,7 @@ proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # Parameter list plist = node.anyChildInTree("parameter_list") - # node[1] could have nested pointers + # node could have nested pointers tcount = node.getPtrCount() # Name could be nested pointer to function diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 3f46849..75e9446 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -27,7 +27,7 @@ struct A4 { float f1; }; -typedef char *A9p[3]; //, A9[4]; +typedef char *A9p[3], A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; typedef struct A0 *A111[12]; @@ -113,6 +113,16 @@ void (*pcre_free)(void *), *(*pcre_stack_malloc)(size_t); +typedef int ImageView, MagickBooleanType; +typedef MagickBooleanType + (*DuplexTransferImageViewMethod)(const ImageView *,const ImageView *, + ImageView *,const size_t,const int,void *), + (*GetImageViewMethod)(const ImageView *,const size_t,const int,void *), + (*SetImageViewMethod)(ImageView *,const size_t,const int,void *), + (*TransferImageViewMethod)(const ImageView *,ImageView *,const size_t, + const int,void *), +(*UpdateImageViewMethod)(ImageView *,const size_t,const int,void *); + @@ -145,12 +155,15 @@ struct A4 { float f1; }; -typedef char *A9p[3]; //, A9[4]; +typedef char *A9p[3], A9[4]; typedef char *A10[3][6]; typedef char *(*A11)[3]; -typedef struct A1 *A111[12]; +typedef struct A0 *A111[12]; -typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int + **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), + **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); typedef int A13(int, int, void (*func)(void)); struct A14 { volatile char a1; }; @@ -222,6 +235,22 @@ typedef enum VSPresetFormat { //struct A2 test_proc1(struct A0 a); +// Proc vars +void + *(*pcre_malloc)(size_t), + (*pcre_free)(void *), + *(*pcre_stack_malloc)(size_t); + +typedef MagickBooleanType + (*DuplexTransferImageViewMethod)(const ImageView *,const ImageView *, + ImageView *,const size_t,const int,void *), + (*GetImageViewMethod)(const ImageView *,const size_t,const int,void *), + (*SetImageViewMethod)(ImageView *,const size_t,const int,void *), + (*TransferImageViewMethod)(const ImageView *,ImageView *,const size_t, + const int,void *), +(*UpdateImageViewMethod)(ImageView *,const size_t,const int,void *); + + #endif #ifdef __cplusplus diff --git a/tests/tast2.nim b/tests/tast2.nim index 28ea7c5..4e60172 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -165,10 +165,10 @@ var a9p: A9p a9p[1] = nil a9p[2] = "hello".cstring -#Not implemented yet -#assert A9 is array[4, cchar] -#checkPragmas(A9, pHeaderImp) -#var a9: A9 +assert A9 is array[4, cchar] +checkPragmas(A9, pHeaderImp) +var a9: A9 +a9[2] = 'c' assert A10 is array[3, array[6, cstring]] checkPragmas(A10, pHeaderImp) @@ -308,4 +308,20 @@ assert pcre_free is proc(a1: pointer) {.cdecl.} checkPragmas(pcre_free, @["importc", "cdecl"] & pHeader) assert pcre_stack_malloc is proc(a1: uint): pointer {.cdecl.} -checkPragmas(pcre_stack_malloc, @["importc", "cdecl"] & pHeader) \ No newline at end of file +checkPragmas(pcre_stack_malloc, @["importc", "cdecl"] & pHeader) + +assert DuplexTransferImageViewMethod is + proc (a1: ptr ImageView; a2: ptr ImageView; a3: ptr ImageView; a4: uint; + a5: cint; a6: pointer): MagickBooleanType {.cdecl.} + +assert GetImageViewMethod is + proc (a1: ptr ImageView; a2: uint; a3: cint; a4: pointer): MagickBooleanType {.cdecl.} + +assert SetImageViewMethod is + proc (a1: ptr ImageView; a2: uint; a3: cint; a4: pointer): MagickBooleanType {.cdecl.} + +assert TransferImageViewMethod is + proc (a1: ptr ImageView; a2: ptr ImageView; a3: uint; a4: cint; a5: pointer): MagickBooleanType {.cdecl.} + +assert UpdateImageViewMethod is + proc (a1: ptr ImageView; a2: uint; a3: cint; a4: pointer): MagickBooleanType {.cdecl.} \ No newline at end of file From e1c8ef2775fda98defbb7f70b976538f24af3b6d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 6 Apr 2020 12:44:07 -0500 Subject: [PATCH 027/255] ast2 dynlib fix, add 1.2.0 testing --- .travis.yml | 1 + nimterop.nimble | 3 +-- nimterop/ast2.nim | 7 +++++++ nimterop/toast.nim | 4 ++++ tests/tsoloud.nim | 3 ++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3201c6c..3b66a7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 + - BRANCH=1.2.0 - BRANCH=devel cache: diff --git a/nimterop.nimble b/nimterop.nimble index be76135..870322b 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -49,8 +49,7 @@ task test, "Test": execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h" execTest "tests/tpcre.nim" - #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" - #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" # Platform specific tests when defined(Windows): diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 7becd04..a6610cd 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -386,6 +386,9 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra if nimState.includeHeader(): # Add header nimState.addPragma(node, prident[1], nimState.impShort & "H") + elif nimState.gState.dynlib.nBl: + # Add dynlib + nimState.addPragma(node, prident[1], "dynlib", nimState.getIdent(nimState.gState.dynlib)) else: # Only need impShort since no name change prident = nimState.newPragmaExpr(node, ident, nimState.impShort) @@ -1457,6 +1460,10 @@ proc addProc(nimState: NimState, node, rnode: TSNode) = # {.convention.} nimState.addPragma(node, prident, nimState.gState.convention) + if nimState.gState.dynlib.nBl: + # {.dynlib.} for DLLs + nimState.addPragma(node, prident, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + procDef.add prident procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index a82a4ea..fe63fb9 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -100,6 +100,10 @@ proc main( symOverride: symOverride ) + # Fail if both includeHeader and dynlib + doAssert not (includeHeader == true and dynlib.nBl), + "`includeHeader` and `dynlib` cannot be used simultaneously" + # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index 9823531..afff0f1 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -43,7 +43,8 @@ cCompile(src/"audiosource", "cpp", exclude="ay/") cCompile(src/"audiosource", "c") cCompile(src/"filter/*.cpp") -cImport(incl/"soloud_c.h") +const FLAGS {.strdefine.} = "" +cImport(incl/"soloud_c.h", flags = FLAGS) var s = Soloud_create() From 4c513a50de64d7675b652e08deeb4c7c0db4124a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 6 Apr 2020 22:38:12 -0500 Subject: [PATCH 028/255] ast2 ignore parseString warnings, optional getNimExpression, handle override nil, param comments --- nimterop.nimble | 4 +++- nimterop/ast2.nim | 38 +++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 870322b..c296ab5 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -29,7 +29,7 @@ task bt, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") task btd, "build toast": - execCmd("nim c --hints:off nimterop/toast.nim") + execCmd("nim c -g --hints:off nimterop/toast.nim") task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") @@ -56,6 +56,8 @@ task test, "Test": execTest "tests/tmath.nim" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): execTest "tests/tsoloud.nim" + execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2\"" + execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2 -H\"" # getHeader tests withDir("tests"): diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index a6610cd..09f5e9d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -2,7 +2,7 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, lineinfos, modulegraphs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] import "."/treesitter/api @@ -18,8 +18,9 @@ proc getPtrType*(str: string): string = str proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = - # Raise exception in parseString() instead of exiting - raise newException(Exception, "") + # Raise exception in parseString() instead of exiting for errors + if msg < warnMin: + raise newException(Exception, msgKindToString(msg)) proc parseString(nimState: NimState, str: string): PNode = # Parse a string into Nim AST - use custom error handler that raises @@ -29,11 +30,13 @@ proc parseString(nimState: NimState, str: string): PNode = str, nimState.identCache, nimState.config, errorHandler = handleError ) except: - discard + decho getCurrentExceptionMsg() -proc getLit*(nimState: NimState, str: string): PNode = +proc getLit*(nimState: NimState, str: string, expression = false): PNode = # Used to convert #define literals into const and expressions # in array sizes + # + # `expression` is true when `str` should be converted into a Nim expression let str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() @@ -54,7 +57,11 @@ proc getLit*(nimState: NimState, str: string): PNode = result = newStrNode(nkStrLit, str[1 .. ^2]) else: - result = nimState.parseString(nimState.getNimExpression(str)) + let + str = + if expression: nimState.getNimExpression(str) + else: str + result = nimState.parseString(str) if result.isNil: result = newNode(nkNilLit) @@ -75,7 +82,10 @@ proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: if override.nBl: if kind == nskProc: skind = "" - result = nimState.parseString(skind & override.replace(origname, name))[0][0] + let + pnode = nimState.parseString(skind & override.replace(origname, name)) + if not pnode.isNil: + result = pnode[0][0] else: necho &"\n# $1'{origname}' skipped" % skind if nimState.gState.debug: @@ -596,11 +606,12 @@ proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode if not node.isNil: for i in 0 ..< node.len: - # Add nkIdentDefs for each param - let - param = nimState.newIdentDefs(name, node[i], i, exported = false) - if not param.isNil: - result.add param + if node[i].getName() == "parameter_declaration": + # Add nkIdentDefs for each param + let + param = nimState.newIdentDefs(name, node[i], i, exported = false) + if not param.isNil: + result.add param proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkProcTy tree for specified proc type @@ -883,7 +894,8 @@ proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode for i in 0 ..< acount: let - size = nimState.getLit(nimState.getNodeVal(cnode[1])) + # Size of array could be a Nim expression + size = nimState.getLit(nimState.getNodeVal(cnode[1]), expression = true) if size.kind != nkNilLit: result = nimState.newArrayTree(cnode, result, size) cnode = cnode[0] From 4ccde62d4f3ed699832c845fde83598027820d8e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 7 Apr 2020 17:40:59 -0500 Subject: [PATCH 029/255] ast2 fix issue 156 - abstract function pointer --- nimterop.nimble | 2 +- nimterop/ast2.nim | 29 +++++++++++++++++++++++++---- tests/include/tast2.h | 24 ++++++++++++++++++++++++ tests/tast2.nim | 12 +++++++++++- tests/tmath.nim | 3 ++- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index c296ab5..58a99b0 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -10,7 +10,7 @@ installDirs = @["nimterop"] installFiles = @["config.nims"] # Dependencies -requires "nim >= 0.20.2", "regex >= 0.13.1", "cligen >= 0.9.43" +requires "nim >= 0.20.2", "regex#v0.13.1", "cligen >= 0.9.43" import nimterop/docs diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 09f5e9d..7a4b70a 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -531,10 +531,11 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn result = nil else: let - fdecl = node[start+1].anyChildInTree("function_declarator") - adecl = node[start+1].anyChildInTree("array_declarator") + fdecl = node[start+1].firstChildInTree("function_declarator") + afdecl = node[start+1].firstChildInTree("abstract_function_declarator") + adecl = node[start+1].firstChildInTree("array_declarator") abst = node[start+1].getName() == "abstract_pointer_declarator" - if fdecl.isNil and adecl.isNil: + if fdecl.isNil and afdecl.isNil and adecl.isNil: if abst: # Only for proc with no named param with pointer type # Create a param name based on offset @@ -577,6 +578,18 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn result.add pident result.add nimState.getTypeProc(name, node[start+1], node[start]) result.add newNode(nkEmpty) + elif not afdecl.isNil: + # Only for proc with no named param with function pointer type + # Create a param name based on offset + # + # int func(int (*)(int *)); + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported) + procTy = nimState.getTypeProc(name, node[start+1], node[start]) + result.add pident + result.add procTy + result.add newNode(nkEmpty) elif not adecl.isNil: # Named param, array type let @@ -963,6 +976,9 @@ proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # node could have nested pointers tcount = node.getPtrCount() + # Nameless function pointer + afdecl = node.firstChildInTree("abstract_function_declarator") + # Name could be nested pointer to function # # (.. @@ -976,7 +992,12 @@ proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # ) # ) # ) - ncount = node.getAtom().tsNodeParent().getPtrCount(reverse = true) + ncount = + if not afdecl.isNil: + # Pointer to function pointer + afdecl[0].getXCount("abstract_pointer_declarator") + else: + node.getAtom().tsNodeParent().getPtrCount(reverse = true) # Return type var diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 75e9446..348e225 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -123,6 +123,17 @@ typedef MagickBooleanType const int,void *), (*UpdateImageViewMethod)(ImageView *,const size_t,const int,void *); +// Issue #156, math.h +void + *absfunptr1 (int (*)(struct A0 *)), + **absfunptr2 (int (**)(struct A1 *)), + absfunptr3 (int *(*)(struct A2 *)), + *absfunptr4 (int *(**)(struct A3 *)), + absfunptr5 (int (*a)(A4 *)); + +int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); + + @@ -241,6 +252,7 @@ void (*pcre_free)(void *), *(*pcre_stack_malloc)(size_t); +typedef int ImageView, MagickBooleanType; typedef MagickBooleanType (*DuplexTransferImageViewMethod)(const ImageView *,const ImageView *, ImageView *,const size_t,const int,void *), @@ -250,6 +262,18 @@ typedef MagickBooleanType const int,void *), (*UpdateImageViewMethod)(ImageView *,const size_t,const int,void *); +// Issue #156, math.h +void + *absfunptr1 (int (*)(struct A0 *)), + **absfunptr2 (int (**)(struct A1 *)), + absfunptr3 (int *(*)(struct A2 *)), + *absfunptr4 (int *(**)(struct A3 *)), + absfunptr5 (int (*a)(A4 *)); + +int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); + + + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index 4e60172..d1b0f26 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -324,4 +324,14 @@ assert TransferImageViewMethod is proc (a1: ptr ImageView; a2: ptr ImageView; a3: uint; a4: cint; a5: pointer): MagickBooleanType {.cdecl.} assert UpdateImageViewMethod is - proc (a1: ptr ImageView; a2: uint; a3: cint; a4: pointer): MagickBooleanType {.cdecl.} \ No newline at end of file + proc (a1: ptr ImageView; a2: uint; a3: cint; a4: pointer): MagickBooleanType {.cdecl.} + +# Issue #156, math.h +assert absfunptr1 is proc(a1: proc(a1: ptr A0): cint {.cdecl.}): pointer {.cdecl.} +assert absfunptr2 is proc(a1: ptr proc(a1: ptr A1): cint {.cdecl.}): ptr pointer {.cdecl.} +assert absfunptr3 is proc(a1: proc(a1: ptr A2): ptr cint {.cdecl.}) {.cdecl.} +assert absfunptr4 is proc(a1: ptr proc(a1: ptr A3): ptr cint {.cdecl.}): pointer {.cdecl.} +assert absfunptr5 is proc(a1: proc(a1: ptr A4): cint {.cdecl.}) {.cdecl.} + +assert sqlite3_bind_blob is + proc(a1: ptr A1, a2: cint, a3: pointer, n: cint, a5: proc(a1: pointer) {.cdecl.}): cint {.cdecl.} diff --git a/tests/tmath.nim b/tests/tmath.nim index 9f3df79..1f804ab 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -22,7 +22,8 @@ cPlugin: proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = sym.name = sym.name.strip(chars={'_'}).replace("__", "_") -cImport cSearchPath("math.h") +const FLAGS {.strdefine.} = "" +cImport(cSearchPath("math.h"), flags = FLAGS) check sin(5) == -0.9589242746631385 check abs(-5) == 5 From 31ed046e3b1d472c70959636114c737345197a44 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 7 Apr 2020 17:54:17 -0500 Subject: [PATCH 030/255] ast2 tmath tests --- nimterop.nimble | 2 ++ tests/tmath.nim | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 58a99b0..cd36e1c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -54,6 +54,8 @@ task test, "Test": # Platform specific tests when defined(Windows): execTest "tests/tmath.nim" + execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2\"" + execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2 -H\"" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): execTest "tests/tsoloud.nim" execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2\"" diff --git a/tests/tmath.nim b/tests/tmath.nim index 1f804ab..5d84700 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -1,10 +1,11 @@ import unittest import nimterop/cimport -type - locale_t = object - mingw_ldbl_type_t = object - mingw_dbl_type_t = object +cOverride: + type + locale_t = object + mingw_ldbl_type_t = object + mingw_dbl_type_t = object when defined(windows): cOverride: From d877b204077c80422666e2f89418cdda64f0720f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 7 Apr 2020 20:24:04 -0500 Subject: [PATCH 031/255] ast2 fix issue 174 - UncheckedArray --- nimterop/ast2.nim | 30 ++++++++++++++++++++++-------- tests/include/tast2.h | 10 ++++++++++ tests/tast2.nim | 12 ++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 7a4b70a..a9624ac 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -460,11 +460,18 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = parent.add result result = nresult -proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = +proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input + # + # If `size` is nil, create UncheckedArray[typ] let info = nimState.getLineInfo(node.getAtom()) - ident = nimState.getIdent("array", info, exported = false) + tname = + if size.isNil: + "UncheckedArray" + else: + "array" + ident = nimState.getIdent(tname, info, exported = false) # array[size, typ] # @@ -475,7 +482,8 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # ) result = newNode(nkBracketExpr) result.add ident - result.add size + if not size.isNil: + result.add size result.add typ proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode @@ -906,11 +914,17 @@ proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode result = nimState.newPtrTree(tcount, result) for i in 0 ..< acount: - let - # Size of array could be a Nim expression - size = nimState.getLit(nimState.getNodeVal(cnode[1]), expression = true) - if size.kind != nkNilLit: - result = nimState.newArrayTree(cnode, result, size) + if cnode.len == 2: + # type name[X] => array[X, type] + let + # Size of array could be a Nim expression + size = nimState.getLit(nimState.getNodeVal(cnode[1]), expression = true) + if size.kind != nkNilLit: + result = nimState.newArrayTree(cnode, result, size) + cnode = cnode[0] + elif cnode.len == 1: + # type name[] = UncheckedArray[type] + result = nimState.newArrayTree(cnode, result) cnode = cnode[0] if ncount > 0: diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 348e225..a819f5e 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -133,6 +133,16 @@ void int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); +// Issue #174 - type name[] => UncheckedArray[type] +int ucArrFunc1(int text[]); +int ucArrFunc2(int text[][5], int (*func)(int text[])); + +typedef int ucArrType1[][5]; +struct ucArrType2 { + float f1[5][5]; + int *f2[][5]; +}; + diff --git a/tests/tast2.nim b/tests/tast2.nim index d1b0f26..f2816eb 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -335,3 +335,15 @@ assert absfunptr5 is proc(a1: proc(a1: ptr A4): cint {.cdecl.}) {.cdecl.} assert sqlite3_bind_blob is proc(a1: ptr A1, a2: cint, a3: pointer, n: cint, a5: proc(a1: pointer) {.cdecl.}): cint {.cdecl.} + +# Issue #174 - type name[] => UncheckedArray[type] +assert ucArrFunc1 is proc(text: UncheckedArray[cint]): cint {.cdecl.} +assert ucArrFunc2 is + proc(text: UncheckedArray[array[5, cint]], `func`: proc(text: UncheckedArray[cint]): cint {.cdecl.}): cint {.cdecl.} + +assert ucArrType1 is UncheckedArray[array[5, cint]] +checkPragmas(ucArrType1, pHeaderImp) + +assert ucArrType2 is object +testFields(ucArrType2, "f1|f2:array[5, array[5, cfloat]]|UncheckedArray[array[5, ptr cint]]") +checkPragmas(ucArrType2, pHeaderBy, istype = false) From 058261c2037d277436d1dc64b53e669f90f0db0e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 8 Apr 2020 17:21:23 -0500 Subject: [PATCH 032/255] ast2 tests and const bugfix --- nimterop/ast2.nim | 8 +++++++- tests/include/tast2.h | 22 ++++++++++++++++++++++ tests/tast2.nim | 38 ++++++++++++++++++++++++-------------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index a9624ac..7bb4520 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -283,7 +283,13 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra # If `pragmas`, add as nkPragmaExpr but not for `nskProc` since procs add pragmas elsewhere # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} when includeHeader let - (tname, torigname, info) = nimState.getNameInfo(node.getAtom(), kind) + atom = node.getAtom() + + (tname, torigname, info) = + if not atom.isNil: + nimState.getNameInfo(node.getAtom(), kind) + else: + ("", "", nimState.getLineInfo(node)) origname = if fname.nBl: diff --git a/tests/include/tast2.h b/tests/include/tast2.h index a819f5e..f4cd201 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -143,6 +143,17 @@ struct ucArrType2 { int *f2[][5]; }; +typedef struct fieldfuncfunc { + int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1))); +}; + +int *func2(int f1, int *(*sfunc2)(int f1, int *(*ssfunc2)(int f1))); + +typedef struct { + const char *name; // description + const char *driver; // driver + int flags; +} BASS_DEVICEINFO; @@ -282,6 +293,17 @@ void int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); +typedef struct fieldfuncfunc { + int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1))); +}; + +int *func2(int f1, int *(*sfunc2)(int f1, int *(*ssfunc2)(int f1))); + +typedef struct { + const char *name; // description + const char *driver; // driver + int flags; +} BASS_DEVICEINFO; diff --git a/tests/tast2.nim b/tests/tast2.nim index f2816eb..a88c29e 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -78,7 +78,7 @@ macro testFields(t: typed, fields: static[string] = "") = var ast = t.getImpl() rl = ast.getRecList() - fsplit = fields.split(":") + fsplit = fields.split("!") names = fsplit[0].split("|") types = if fsplit.len > 1: @@ -89,7 +89,7 @@ macro testFields(t: typed, fields: static[string] = "") = for i in 0 ..< rl.len: let name = ($rl[i][0]).strip(chars = {'*'}) - typ = $(rl[i][1].repr()) + typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "") n = names.find(name) assert n != -1, $t & "." & name & " invalid" assert types[n] == typ, @@ -102,13 +102,13 @@ assert D == "hello" assert E == 'c' assert A0 is object -testFields(A0, "f1:cint") +testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) var a0: A0 a0.f1 = 1 assert A1 is A0 -testFields(A1, "f1:cint") +testFields(A1, "f1!cint") var a1: A1 a1.f1 = 2 @@ -125,13 +125,13 @@ checkPragmas(A3, pHeaderInc, istype = false) var a3: A3 assert A4 is object -testFields(A4, "f1:cfloat") +testFields(A4, "f1!cfloat") checkPragmas(A4, pHeaderImpBy) var a4: A4 a4.f1 = 4.1 assert A4p is ptr A4 -testFields(A4p, "f1:cfloat") +testFields(A4p, "f1!cfloat") checkPragmas(A4p, pHeaderImp) var a4p: A4p a4p = addr a4 @@ -202,13 +202,13 @@ checkPragmas(A13, pHeaderImp & "cdecl") var a13: A13 assert A14 is object -testFields(A14, "a1:cchar") +testFields(A14, "a1!cchar") checkPragmas(A14, pHeaderBy, istype = false) var a14: A14 a14.a1 = 'a' assert A15 is object -testFields(A15, "a1|a2:cstring|array[1, ptr cint]") +testFields(A15, "a1|a2!cstring|array[1, ptr cint]") checkPragmas(A15, pHeaderBy, istype = false) var a15: A15 @@ -217,13 +217,13 @@ a15.a1 = "hello".cstring a15.a2[0] = addr a15i assert A16 is object -testFields(A16, "f1:cchar") +testFields(A16, "f1!cchar") checkPragmas(A16, pHeaderBy, istype = false) var a16: A16 a16.f1 = 's' assert A17 is object -testFields(A17, "a1|a2:cstring|array[1, ptr cint]") +testFields(A17, "a1|a2!cstring|array[1, ptr cint]") checkPragmas(A17, pHeaderBy, istype = false) var a17: A17 a17.a1 = "hello".cstring @@ -239,7 +239,7 @@ var a18p: A18p a18p = addr a18 assert A19 is object -testFields(A19, "a1|a2:cstring|array[1, ptr cint]") +testFields(A19, "a1|a2!cstring|array[1, ptr cint]") checkPragmas(A19, pHeaderImpBy) var a19: A19 a19.a1 = "hello".cstring @@ -251,7 +251,7 @@ var a19p: A19p a19p = addr a19 assert A20 is object -testFields(A20, "a1:cchar") +testFields(A20, "a1!cchar") checkPragmas(A20, pHeaderBy, istype = false) var a20: A20 a20.a1 = 'a' @@ -267,7 +267,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2:ptr ptr cint|array[123 + 132, ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[123 + 132, ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] @@ -345,5 +345,15 @@ assert ucArrType1 is UncheckedArray[array[5, cint]] checkPragmas(ucArrType1, pHeaderImp) assert ucArrType2 is object -testFields(ucArrType2, "f1|f2:array[5, array[5, cfloat]]|UncheckedArray[array[5, ptr cint]]") +testFields(ucArrType2, "f1|f2!array[5, array[5, cfloat]]|UncheckedArray[array[5, ptr cint]]") checkPragmas(ucArrType2, pHeaderBy, istype = false) + +assert fieldfuncfunc is object +testFields(fieldfuncfunc, + "func1!proc (f1: cint; sfunc1: proc (f1: cint; ssfunc1: proc (f1: cint): ptr cint {.cdecl.}): ptr cint {.cdecl.}): ptr cint {.cdecl.}") + +assert func2 is proc (f1: cint; sfunc2: proc (f1: cint; ssfunc2: proc (f1: cint): ptr cint {.cdecl.}): ptr cint {.cdecl.}): ptr cint {.cdecl.} + +assert BASS_DEVICEINFO is object +testFields(BASS_DEVICEINFO, "name|driver|flags!cstring|cstring|cint") +checkPragmas(BASS_DEVICEINFO, pHeaderImpBy) \ No newline at end of file From f5dd89904d8866456c62f4bf6b57b6c264d6dea2 Mon Sep 17 00:00:00 2001 From: awr1 <41453959+awr1@users.noreply.github.com> Date: Thu, 9 Apr 2020 23:00:56 -0500 Subject: [PATCH 033/255] Preprocessor mode fixes (#176) --- nimterop/build.nim | 38 ++++++++++++++++++++++++++------------ nimterop/getters.nim | 5 ++--- nimterop/globals.nim | 9 ++------- nimterop/toast.nim | 13 ++++--------- tests/tpcre.nim | 3 +-- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index c65ec2a..cc6dc22 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -620,6 +620,21 @@ proc make*(path, check: string, flags = "", regex = false) = doAssert findFile(check, path, regex = regex).len != 0, "# make failed" +proc getCompilerMode*(path: string): string = + ## Determines a target language mode from an input filename, if one is not already specified. + let file = path.splitFile() + if file.ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: + result = "cpp" + elif file.ext in [".h", ".c"]: + result = "c" + +proc getGccModeArg*(mode: string): string = + ## Produces a GCC argument that explicitly sets the language mode to be used by the compiler. + if mode == "cpp": + result = "-xc++" + elif mode == "c": + result = "-xc" + proc getCompiler*(): string = var compiler = @@ -632,13 +647,12 @@ proc getCompiler*(): string = result = getEnv("CC", compiler) -proc getGccPaths*(mode = "c"): seq[string] = +proc getGccPaths*(mode: string): seq[string] = var nul = when defined(Windows): "nul" else: "/dev/null" - mmode = if mode == "cpp": "c++" else: mode inc = false - (outp, _) = execAction(&"""{getCompiler()} -Wp,-v -x{mmode} {nul}""", die = false) + (outp, _) = execAction(&"""{getCompiler()} -Wp,-v {getGccModeArg(mode)} {nul}""", die = false) for line in outp.splitLines(): if "#include <...> search starts here" in line: @@ -655,13 +669,12 @@ proc getGccPaths*(mode = "c"): seq[string] = when defined(osx): result.add(execAction("xcrun --show-sdk-path").output.strip() & "/usr/include") -proc getGccLibPaths*(mode = "c"): seq[string] = +proc getGccLibPaths*(mode: string): seq[string] = var nul = when defined(Windows): "nul" else: "/dev/null" - mmode = if mode == "cpp": "c++" else: mode linker = when defined(OSX): "-Xlinker" else: "" - (outp, _) = execAction(&"""{getCompiler()} {linker} -v -x{mmode} {nul}""", die = false) + (outp, _) = execAction(&"""{getCompiler()} {linker} -v {getGccModeArg(mode)} {nul}""", die = false) for line in outp.splitLines(): if "LIBRARY_PATH=" in line: @@ -680,14 +693,14 @@ proc getGccLibPaths*(mode = "c"): seq[string] = when defined(osx): result.add "/usr/lib" -proc getStdPath(header: string): string = - for inc in getGccPaths(): +proc getStdPath(header, mode: string): string = + for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) if result.len != 0: break -proc getStdLibPath(lname: string): string = - for lib in getGccLibPaths(): +proc getStdLibPath(lname, mode: string): string = + for lib in getGccLibPaths(mode): result = findFile(lname, lib, recurse = false, first = true, regex = true) if result.len != 0: break @@ -973,6 +986,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta gDefines[verStr] else: "" + mode = getCompilerMode(header) # Use alternate library names if specified for regex search if altNames.len != 0: @@ -1008,9 +1022,9 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta # Look in standard path if requested by user stdPath = - when `nameStd`: getStdPath(`header`) else: "" + when `nameStd`: getStdPath(`header`, `mode`) else: "" stdLPath = - when `nameStd`: getStdLibPath(`lname`) else: "" + when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" # Look elsewhere if requested while prioritizing standard paths prePath = diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 044cde7..986c54d 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -495,11 +495,10 @@ proc removeStatic(content: string): string = result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) ) -proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string = +proc getPreprocessor*(gState: State, fullpath: string): string = var - mmode = if mode == "cpp": "c++" else: mode cmts = if gState.nocomments: "" else: "-CC" - cmd = &"""{getCompiler()} -E {cmts} -dD -x{mmode} -w """ + cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ rdata: seq[string] = @[] start = false diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 701db61..42f7cb7 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -98,9 +98,6 @@ type nodeBranch*: seq[string] - CompileMode = enum - c, cpp - Feature* = enum ast2 @@ -113,11 +110,9 @@ template nBl(s: typed): untyped {.used.} = template Bl(s: typed): untyped {.used.} = (s.len == 0) -const modeDefault {.used.} = $cpp - when not declared(CIMPORT): export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, - nBl, Bl, CompileMode, modeDefault + nBl, Bl # Redirect output to file when required template gecho*(args: string) {.dirty.} = @@ -133,4 +128,4 @@ when not declared(CIMPORT): template decho*(str: untyped): untyped = if nimState.gState.debug: - necho str.getCommented() \ No newline at end of file + necho str.getCommented() diff --git a/nimterop/toast.nim b/nimterop/toast.nim index fe63fb9..c3d6a3d 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,23 +2,18 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, globals, getters, grammar] +import "."/[ast, ast2, globals, getters, grammar, build] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" - var - parser = tsParserNew() - ext = path.splitFile().ext + var parser = tsParserNew() defer: parser.tsParserDelete() if gState.mode.Bl: - if ext in [".h", ".c"]: - gState.mode = "c" - elif ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: - gState.mode = "cpp" + gState.mode = getCompilerMode(path) if gState.preprocess: gState.code = gState.getPreprocessor(path) @@ -61,7 +56,7 @@ proc main( feature: seq[Feature] = @[], includeHeader = false, includeDirs: seq[string] = @[], - mode = modeDefault, + mode = "", nim: string = "nim", nocomments = false, output = "", diff --git a/tests/tpcre.nim b/tests/tpcre.nim index f896e9c..c8e8059 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -31,8 +31,7 @@ cPlugin: sym.name = sym.name.replace("pcre_", "") const FLAGS {.strdefine.} = "" -cImport(pcreH, dynlib="dynpcre", flags = FLAGS) - +cImport(pcreH, dynlib="dynpcre", flags="--mode=c " & FLAGS) echo version() when FLAGS.len != 0: From f64fbb67b11ed6e48bf1585d14c1fded3c173944 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 10 Apr 2020 22:45:29 -0500 Subject: [PATCH 034/255] Fix #178 - single underscore --- nimterop/getters.nim | 4 +++- tests/include/test.h | 6 ++++++ tests/tnimterop_c.nim | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 986c54d..48ed8f7 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -655,7 +655,9 @@ proc getNimExpression*(nimState: NimState, expr: string, name = ""): string = if i == clean.len or gen.nBl: # Process identifier if ident.nBl: - ident = nimState.getIdentifier(ident, nskConst) + # Issue #178 + if ident != "_": + ident = nimState.getIdentifier(ident, nskConst) if name.nBl and ident in nimState.constIdentifiers: ident = ident & "." & name result &= ident diff --git a/tests/include/test.h b/tests/include/test.h index b22e2d8..e9c281c 100644 --- a/tests/include/test.h +++ b/tests/include/test.h @@ -194,6 +194,12 @@ struct JKL { int **f1; }; +// Issue #178 +typedef enum +{ + SDLK_UNDERSCORE = '_' +} SDL_KeyCode; + #ifdef __cplusplus } #endif diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index fe4e9ee..e78e7f2 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -227,3 +227,4 @@ cg.f2 = nil doAssert BIT == 1 doAssert ca(nil) == 1 doAssert cc(nil) == 2 +doAssert SDLK_UNDERSCORE == 95 \ No newline at end of file From 894193eb43b81267f83bc722204d33df28478f79 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 10 Apr 2020 23:49:23 -0500 Subject: [PATCH 035/255] Fix type override issue --- nimterop/grammar.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index 2e6b585..8e1d290 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -178,7 +178,7 @@ proc initGrammar(): Grammar = var i = 0 - typ = nimState.getIdentifier(nimState.data[i].val, nskType).getType() + typ = nimState.getIdentifier(nimState.data[i].val, nskType, "Parent").getType() name = "" nname = "" tptr = "" From ecee58342ee53296eae98fcf4038e6f999b416a5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 11 Apr 2020 14:54:19 -0500 Subject: [PATCH 036/255] Fix expression override issue --- nimterop/getters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 48ed8f7..e47113d 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -657,7 +657,7 @@ proc getNimExpression*(nimState: NimState, expr: string, name = ""): string = if ident.nBl: # Issue #178 if ident != "_": - ident = nimState.getIdentifier(ident, nskConst) + ident = nimState.getIdentifier(ident, nskConst, name) if name.nBl and ident in nimState.constIdentifiers: ident = ident & "." & name result &= ident From 9cd39600d43c90045b0da6873900881449e6d609 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Apr 2020 12:19:13 -0500 Subject: [PATCH 037/255] Partly fix issue #183, lineTrace:on --- config.nims | 1 + nimterop/ast2.nim | 161 +++++++++++++++++++++++------------------- tests/include/tast2.h | 13 ++++ tests/tast2.nim | 7 +- 4 files changed, 109 insertions(+), 73 deletions(-) diff --git a/config.nims b/config.nims index 7e7a6e1..6b475de 100644 --- a/config.nims +++ b/config.nims @@ -10,6 +10,7 @@ when defined(Windows): # Retain stackTrace for clear errors switch("stackTrace", "on") +switch("lineTrace", "on") # Path to compiler switch("path", "$nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 7bb4520..ed7c2a9 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -495,7 +495,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNo proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode -proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = +iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field # # For proc, param should not be exported @@ -520,8 +520,13 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn # typ, # nkEmpty() # ) - result = newNode(nkIdentDefs) - + # + # Iterator since structs can have multiple comma separated fields for the + # same type so can yield multiple results. + # + # struct ABC { int w, h; }; + # + # This is not applicable for procs. let start = getStartAtom(node) @@ -533,6 +538,9 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn # Only for proc with no named param - create a param name based on offset # # int func(char, int); + var + result = newNode(nkIdentDefs) + if tname != "object": let pname = "a" & $(offset+1) @@ -543,77 +551,88 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn else: # int func(void) result = nil + + yield result else: - let - fdecl = node[start+1].firstChildInTree("function_declarator") - afdecl = node[start+1].firstChildInTree("abstract_function_declarator") - adecl = node[start+1].firstChildInTree("array_declarator") - abst = node[start+1].getName() == "abstract_pointer_declarator" - if fdecl.isNil and afdecl.isNil and adecl.isNil: - if abst: - # Only for proc with no named param with pointer type + for i in start+1 ..< node.len: + if node[i].getName() == "bitfield_clause": + continue + + var + result = newNode(nkIdentDefs) + + let + fdecl = node[i].firstChildInTree("function_declarator") + afdecl = node[i].firstChildInTree("abstract_function_declarator") + adecl = node[i].firstChildInTree("array_declarator") + abst = node[i].getName() == "abstract_pointer_declarator" + if fdecl.isNil and afdecl.isNil and adecl.isNil: + if abst: + # Only for proc with no named param with pointer type + # Create a param name based on offset + # + # int func(char *, int **); + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported) + acount = node[i].getXCount("abstract_pointer_declarator") + result.add pident + result.add nimState.newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + # Named param, simple type + let + (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + + # Bitfield support - typedef struct { int field: 1; }; + prident = + if node.len > i and node[i + 1].getName() == "bitfield_clause": + nimState.newPragmaExpr(node, pident, "bitsize", + newIntNode(nkIntLit, parseInt(nimState.getNodeVal(node[i + 1].getAtom())))) + else: + pident + + count = node[i].getPtrCount() + + result.add prident + if count > 0: + result.add nimState.newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + # Named param, function pointer + let + (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + result.add pident + result.add nimState.getTypeProc(name, node[i], node[start]) + result.add newNode(nkEmpty) + elif not afdecl.isNil: + # Only for proc with no named param with function pointer type # Create a param name based on offset # - # int func(char *, int **); + # int func(int (*)(int *)); let pname = "a" & $(offset+1) pident = nimState.getIdent(pname, tinfo, exported) - acount = node[start+1].getXCount("abstract_pointer_declarator") + procTy = nimState.getTypeProc(name, node[i], node[start]) result.add pident - result.add nimState.newPtrTree(acount, tident) + result.add procTy + result.add newNode(nkEmpty) + elif not adecl.isNil: + # Named param, array type + let + (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported) + result.add pident + result.add nimState.getTypeArray(node[i], node[start], name) result.add newNode(nkEmpty) else: - # Named param, simple type - let - (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) + result = nil - # Bitfield support - typedef struct { int field: 1; }; - prident = - if node.len > start+1 and node[start+2].getName() == "bitfield_clause": - nimState.newPragmaExpr(node, pident, "bitsize", - newIntNode(nkIntLit, parseInt(nimState.getNodeVal(node[start+2].getAtom())))) - else: - pident - - count = node[start+1].getPtrCount() - - result.add prident - if count > 0: - result.add nimState.newPtrTree(count, tident) - else: - result.add tident - result.add newNode(nkEmpty) - elif not fdecl.isNil: - # Named param, function pointer - let - (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) - result.add pident - result.add nimState.getTypeProc(name, node[start+1], node[start]) - result.add newNode(nkEmpty) - elif not afdecl.isNil: - # Only for proc with no named param with function pointer type - # Create a param name based on offset - # - # int func(int (*)(int *)); - let - pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported) - procTy = nimState.getTypeProc(name, node[start+1], node[start]) - result.add pident - result.add procTy - result.add newNode(nkEmpty) - elif not adecl.isNil: - # Named param, array type - let - (pname, _, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) - result.add pident - result.add nimState.getTypeArray(node[start+1], node[start], name) - result.add newNode(nkEmpty) - else: - result = nil + yield result proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkFormalParams tree for specified params and return type @@ -635,10 +654,9 @@ proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode for i in 0 ..< node.len: if node[i].getName() == "parameter_declaration": # Add nkIdentDefs for each param - let - param = nimState.newIdentDefs(name, node[i], i, exported = false) - if not param.isNil: - result.add param + for param in nimState.newIdentDefs(name, node[i], i, exported = false): + if not param.isNil: + result.add param proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkProcTy tree for specified proc type @@ -674,10 +692,9 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = for i in 0 ..< node.len: if node[i].getName() == "field_declaration": # Add nkIdentDefs for each field - let - field = nimState.newIdentDefs(name, node[i], i, exported = true) - if not field.isNil: - result.add field + for field in nimState.newIdentDefs(name, node[i], i, exported = true): + if not field.isNil: + result.add field proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object diff --git a/tests/include/tast2.h b/tests/include/tast2.h index f4cd201..4854eb7 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -155,6 +155,12 @@ typedef struct { int flags; } BASS_DEVICEINFO; +// Issue #183 +struct GPU_Target +{ + int w, *h; + char *x, y, **z; +}; @@ -305,6 +311,13 @@ typedef struct { int flags; } BASS_DEVICEINFO; +// Issue #183 +struct GPU_Target +{ + int w, *h; + char *x, y, **z; +}; + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index a88c29e..de0e743 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -356,4 +356,9 @@ assert func2 is proc (f1: cint; sfunc2: proc (f1: cint; ssfunc2: proc (f1: cint) assert BASS_DEVICEINFO is object testFields(BASS_DEVICEINFO, "name|driver|flags!cstring|cstring|cint") -checkPragmas(BASS_DEVICEINFO, pHeaderImpBy) \ No newline at end of file +checkPragmas(BASS_DEVICEINFO, pHeaderImpBy) + +# Issue #183 +assert GPU_Target is object +testFields(GPU_Target, "w|h|x|y|z!cint|ptr cint|cstring|cchar|ptr cstring") +checkPragmas(GPU_Target, pHeaderBy, istype = false) \ No newline at end of file From 0afb634b59d55ff369bd25d85e13d156c6836db0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Apr 2020 12:56:50 -0500 Subject: [PATCH 038/255] Fix #181 - cPluginPath() --- nimterop/cimport.nim | 18 ++++++++++++++++-- tests/tnimterop_c.nim | 9 +-------- tests/tnimterop_c_plugin.nim | 7 +++++++ 3 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 tests/tnimterop_c_plugin.nim diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index c4ebc3f..69aaf90 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -273,7 +273,7 @@ proc cPluginHelper(body: string) = if gStateCT.pluginSource.nBl or gStateCT.overrides.nBl: let - data = "import macros, nimterop/plugin\n\n" & body & gStateCT.overrides + data = "import macros, nimterop/plugin\n\n" & body & "\n\n" & gStateCT.overrides hash = data.hash().abs() path = getProjectCacheDir("cPlugins", forceClean = false) / "nimterop_" & $hash & ".nim" @@ -317,7 +317,7 @@ macro cPlugin*(body): untyped = ## - `nskEnumField` for enum (field) names, though they are in the global namespace as `nskConst` ## - `nskProc` - for proc names ## - ## `nimterop/plugins` is implicitly imported to provide access to standard + ## `macros` and `nimterop/plugins` are implicitly imported to provide access to standard ## plugin facilities. ## ## `cPlugin() `_ only affects calls to @@ -342,6 +342,20 @@ macro cPlugin*(body): untyped = cPluginHelper(body.repr) +macro cPluginPath*(path: static[string]): untyped = + ## Rather than embedding the `cPlugin()` code within the wrapper, it might be + ## preferable to have it stored in a separate source file. This allows for reuse + ## across multiple wrappers when applicable. + ## + ## The `cPluginPath()` macro enables this functionality - provide a path to the + ## plugin file and it will be consumed in the same way as `cPlugin()`. + ## + ## `path` is relative to the current dir and not necessarily relative to the + ## location of the wrapper file. Use `currentSourcePath.parentDir()` to specify + ## path relative to the wrapper file. + doAssert fileExists(path), "Plugin file not found: " & path + cPluginHelper(readFile(path)) + proc cSearchPath*(path: string): string {.compileTime.}= ## Get full path to file or directory `path` in search path configured ## using `cAddSearchDir() `_ and diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index e78e7f2..0360225 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -11,14 +11,7 @@ cDefine("FORCE") cIncludeDir testsIncludeDir() cCompile cSearchPath("test.c") -cPlugin: - import strutils - - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - if sym.name == "_Kernel": - sym.name = "uKernel" - else: - sym.name = sym.name.strip(chars={'_'}) +cPluginPath("tests/tnimterop_c_plugin.nim") cOverride: type diff --git a/tests/tnimterop_c_plugin.nim b/tests/tnimterop_c_plugin.nim new file mode 100644 index 0000000..68bb4d6 --- /dev/null +++ b/tests/tnimterop_c_plugin.nim @@ -0,0 +1,7 @@ +import strutils + +proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = + if sym.name == "_Kernel": + sym.name = "uKernel" + else: + sym.name = sym.name.strip(chars={'_'}) \ No newline at end of file From b2de34328a92c6c9a63e3db12d4f78e7e5831d8c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Apr 2020 23:12:38 -0500 Subject: [PATCH 039/255] No explicit imports for cPluginPath --- nimterop/cimport.nim | 16 ++++++++++------ tests/tnimterop_c_plugin.nim | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 69aaf90..382c034 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -268,12 +268,12 @@ proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = static: cSkipSymbol @["proc1", "Type2"] gStateCT.symOverride.add skips -proc cPluginHelper(body: string) = +proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") = gStateCT.pluginSource = body if gStateCT.pluginSource.nBl or gStateCT.overrides.nBl: let - data = "import macros, nimterop/plugin\n\n" & body & "\n\n" & gStateCT.overrides + data = imports & body & "\n\n" & gStateCT.overrides hash = data.hash().abs() path = getProjectCacheDir("cPlugins", forceClean = false) / "nimterop_" & $hash & ".nim" @@ -347,14 +347,18 @@ macro cPluginPath*(path: static[string]): untyped = ## preferable to have it stored in a separate source file. This allows for reuse ## across multiple wrappers when applicable. ## - ## The `cPluginPath()` macro enables this functionality - provide a path to the + ## The `cPluginPath()` macro enables this functionality - specify the path to the ## plugin file and it will be consumed in the same way as `cPlugin()`. ## ## `path` is relative to the current dir and not necessarily relative to the - ## location of the wrapper file. Use `currentSourcePath.parentDir()` to specify - ## path relative to the wrapper file. + ## location of the wrapper file. Use `currentSourcePath` to specify a path relative + ## to the wrapper file. + ## + ## Unlike `cPlugin()`, this macro also does not implicitly import any other modules + ## since the standalone plugin file will need explicit imports for `nim check` and + ## suggestions to work. `import nimterop/plugin` is required for all plugins. doAssert fileExists(path), "Plugin file not found: " & path - cPluginHelper(readFile(path)) + cPluginHelper(readFile(path), imports = "") proc cSearchPath*(path: string): string {.compileTime.}= ## Get full path to file or directory `path` in search path configured diff --git a/tests/tnimterop_c_plugin.nim b/tests/tnimterop_c_plugin.nim index 68bb4d6..90ac2ab 100644 --- a/tests/tnimterop_c_plugin.nim +++ b/tests/tnimterop_c_plugin.nim @@ -1,3 +1,5 @@ +import nimterop/plugin + import strutils proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = From f23d17d0b2f2e6bd66b0d685461d9dd1c2070918 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 13 Apr 2020 10:18:49 -0500 Subject: [PATCH 040/255] Fix #185 - forward decl symbol change --- nimterop/ast2.nim | 11 +++++++++-- tests/include/tast2.h | 16 ++++++++++++++++ tests/tast2.nim | 16 +++++++++++----- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index ed7c2a9..708b473 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -811,9 +811,16 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if not fdlist.isNil and fdlist.len > 0: # Current node has fields let - name = nimState.getNodeVal(node.getAtom()) + origname = nimState.getNodeVal(node.getAtom()) - if nimState.identifierNodes.hasKey(name): + # Fix issue #185 + name = + if origname.nBl: + nimState.getIdentifier(origname, nskType) + else: + "" + + if name.nBl and nimState.identifierNodes.hasKey(name): let def = nimState.identifierNodes[name] # Duplicate nkTypeDef for `name` with empty fields diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 4854eb7..76aadc9 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -162,6 +162,14 @@ struct GPU_Target char *x, y, **z; }; +// Issue #185 +struct SDL_AudioCVT; + +typedef struct SDL_AudioCVT +{ + int needed; +} SDL_AudioCVT; + // DUPLICATES @@ -318,6 +326,14 @@ struct GPU_Target char *x, y, **z; }; +// Issue #185 +struct SDL_AudioCVT; + +typedef struct SDL_AudioCVT +{ + int needed; +} SDL_AudioCVT; + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index de0e743..4741062 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -32,7 +32,7 @@ cOverride: type A1* = A0 -cImport(path, flags="-f:ast2 -ENK_" & flags) +cImport(path, flags="-f:ast2 -ENK_,SDL_" & flags) proc getPragmas(n: NimNode): HashSet[string] = # Find all pragmas in AST, return as "name" or "name:value" in set @@ -57,7 +57,8 @@ proc getRecList(n: NimNode): NimNode = if not rl.isNil: return rl -macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] = true): untyped = +macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] = true, + prefix: static[string] = ""): untyped = # Verify that type has expected pragmas defined # `istype` is true when typedef X var @@ -67,9 +68,9 @@ macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] when defined(HEADER): if not istype: if "union" in exprag: - exprag.incl "importc:union " & $t + exprag.incl "importc:union " & $prefix & $t else: - exprag.incl "importc:struct " & $t + exprag.incl "importc:struct " & $prefix & $t doAssert symmetricDifference(prag, exprag).len == 0, "\nWrong number of pragmas in " & $t & "\n" & $prag & " vs " & $exprag @@ -361,4 +362,9 @@ checkPragmas(BASS_DEVICEINFO, pHeaderImpBy) # Issue #183 assert GPU_Target is object testFields(GPU_Target, "w|h|x|y|z!cint|ptr cint|cstring|cchar|ptr cstring") -checkPragmas(GPU_Target, pHeaderBy, istype = false) \ No newline at end of file +checkPragmas(GPU_Target, pHeaderBy, istype = false) + +# Issue #185 +assert AudioCVT is object +testFields(AudioCVT, "needed!cint") +checkPragmas(AudioCVT, pHeaderBy, istype = false, "SDL_") \ No newline at end of file From 08306b98020d5dcfd513b515f7955d44cfef1d6b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 13 Apr 2020 20:18:37 -0500 Subject: [PATCH 041/255] Fix #172 - skip qualifiers --- nimterop.nimble | 2 +- nimterop/getters.nim | 18 ++++++++++++++++-- tests/include/tast2.h | 10 ++++++++++ tests/tast2.nim | 7 ++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index cd36e1c..9600faa 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -23,7 +23,7 @@ proc execTest(test: string, flags = "") = execCmd "nim cpp --hints:off " & flags & " -r " & test task buildToast, "build toast": - execCmd("nim c --hints:off -f nimterop/toast.nim") + execCmd("nim c --hints:off nimterop/toast.nim") task bt, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") diff --git a/nimterop/getters.nim b/nimterop/getters.nim index e47113d..76b528d 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -234,7 +234,14 @@ proc getAtom*(node: TSNode): TSNode = if node.getName() in gAtoms: return node elif node.len() != 0: - return node[0].getAtom() + if node[0].getName() == "type_qualifier": + # Skip const, volatile + if node.len() > 1: + return node[1].getAtom() + else: + return + else: + return node[0].getAtom() proc getStartAtom*(node: TSNode): int = if not node.isNil: @@ -256,7 +263,14 @@ proc getXCount*(node: TSNode, ntype: string, reverse = false): int = cnode = cnode.tsNodeParent() else: if cnode.len() != 0: - cnode = cnode[0] + if cnode[0].getName() == "type_qualifier": + # Skip const, volatile + if cnode.len() > 1: + cnode = cnode[1] + else: + break + else: + cnode = cnode[0] else: break diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 76aadc9..136784a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -170,6 +170,11 @@ typedef struct SDL_AudioCVT int needed; } SDL_AudioCVT; +// Issue #172 +typedef struct { + const char* const* x; +} SomeType; + // DUPLICATES @@ -334,6 +339,11 @@ typedef struct SDL_AudioCVT int needed; } SDL_AudioCVT; +// Issue #172 +typedef struct { + const char* const* x; +} SomeType; + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index 4741062..6d749c6 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -367,4 +367,9 @@ checkPragmas(GPU_Target, pHeaderBy, istype = false) # Issue #185 assert AudioCVT is object testFields(AudioCVT, "needed!cint") -checkPragmas(AudioCVT, pHeaderBy, istype = false, "SDL_") \ No newline at end of file +checkPragmas(AudioCVT, pHeaderBy, istype = false, "SDL_") + +# Issue #172 +assert SomeType is object +testFields(SomeType, "x!ptr cstring") +checkPragmas(SomeType, pHeaderImpBy) \ No newline at end of file From 92fe90f834b1afc1098d4ed63760ca811ab48484 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Apr 2020 19:40:46 -0500 Subject: [PATCH 042/255] ast2 nested struct support --- nimterop.nimble | 2 +- nimterop/ast2.nim | 77 +++++++++++++++++++++++++++++++++++++------- nimterop/getters.nim | 28 ++++++++-------- nimterop/lisp.nim | 2 +- 4 files changed, 82 insertions(+), 27 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 9600faa..37c395d 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -29,7 +29,7 @@ task bt, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") task btd, "build toast": - execCmd("nim c -g --hints:off nimterop/toast.nim") + execCmd("nim c -g nimterop/toast.nim") task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 708b473..8dba84a 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -495,10 +495,12 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNo proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode -iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = +iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field # - # For proc, param should not be exported + # For proc, param should not be `exported` + # + # If `ftname` is set, use it as the type name # # pname: [ptr ..] typ # @@ -531,7 +533,15 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So start = getStartAtom(node) # node[start] - param type - (tname, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + + # Override type name + tname = + if ftname.nBl: + ftname + else: + tname0 + tident = nimState.getIdent(tname, tinfo, exported = false) if start == node.len - 1: @@ -676,6 +686,7 @@ proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNo result.add nimState.newFormalParams(name, node, rtyp) result.add nimState.newPragma(node, nimState.gState.convention) +proc processNode(nimState: NimState, node: TSNode): bool proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object if not node.isNil: @@ -691,8 +702,34 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = for i in 0 ..< node.len: if node[i].getName() == "field_declaration": + # Check for nested structs / unions / enums + let + fdecl = node[i].anyChildInTree("field_declaration_list") + edecl = node[i].anyChildInTree("enumerator_list") + + # `tname` is name of nested struct / union / enum just + # added, passed on as type name for field in `newIdentDefs()` + (processed, tname) = + if not fdecl.isNil: + # Nested struct / union + ( + nimState.processNode(fdecl.tsNodeParent()), + nimState.typeSection[^1].getIdentName() + ) + elif not edecl.isNil: + # Nested enum + ( + nimState.processNode(edecl.tsNodeParent()), + $nimState.enumSection[^1][0][1] + ) + else: + (true, "") + + if not processed: + return nil + # Add nkIdentDefs for each field - for field in nimState.newIdentDefs(name, node[i], i, exported = true): + for field in nimState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: result.add field @@ -722,6 +759,16 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname pragmas + fname = + if not node.firstChildInTree("field_declaration_list").isNil and + node.tsNodeParent().getName() == "field_declaration": + # If nested struct / union without a name + nimState.getUniqueIdentifier( + if union: "Union" else: "Type" + ) + else: + fname + typeDefExisting = not typeDef.isNil typeDef = @@ -784,7 +831,11 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if not fdlist.isNil and fdlist.len > 0: # Add fields to object if present - obj.add nimState.newRecListTree(name, fdlist) + let + fields = nimState.newRecListTree(name, fdlist) + if fields.isNil: + return + obj.add fields else: obj.add newNode(nkEmpty) @@ -828,7 +879,11 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname def[2].kind == nkObjectTy and def[2].len == 3 and def[2][2].kind == nkEmpty: # Add fields to existing object - def[2][2] = nimState.newRecListTree(name, fdlist) + let + fields = nimState.newRecListTree(name, fdlist) + if fields.isNil: + return + def[2][2] = fields # Change incompleteStruct to bycopy pragma if def[0].kind == nkPragmaExpr and def[0].len == 2 and @@ -1180,7 +1235,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = let fdecl = node[1].anyChildInTree("function_declarator") adecl = node[1].anyChildInTree("array_declarator") - if fdlist.isNil(): + if fdlist.isNil: if adecl.isNil and fdecl.isNil: # typedef X Y; # typedef X *Y; @@ -1605,18 +1660,18 @@ proc searchTree(nimState: NimState, root: TSNode) = processed = false while true: - if not node.isNil() and depth > -1: + if not node.isNil and depth > -1: processed = nimState.processNode(node) else: break - if not processed and node.len() != 0: + if not processed and node.len != 0: nextnode = node[0] depth += 1 else: nextnode = node.tsNodeNextNamedSibling() - if nextnode.isNil(): + if nextnode.isNil: while true: node = node.tsNodeParent() depth -= 1 @@ -1624,7 +1679,7 @@ proc searchTree(nimState: NimState, root: TSNode) = break if node == root: break - if not node.tsNodeNextNamedSibling().isNil(): + if not node.tsNodeNextNamedSibling().isNil: node = node.tsNodeNextNamedSibling() break else: diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 76b528d..aeefbd8 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -214,7 +214,7 @@ proc len*(node: TSNode): int = result = node.tsNodeNamedChildCount().int proc `[]`*(node: TSNode, i: SomeInteger): TSNode = - if i < node.len(): + if i < node.len: result = node.tsNodeNamedChild(i.uint32) proc getName*(node: TSNode): string {.inline.} = @@ -233,10 +233,10 @@ proc getAtom*(node: TSNode): TSNode = # Get child node which is topmost atom if node.getName() in gAtoms: return node - elif node.len() != 0: + elif node.len != 0: if node[0].getName() == "type_qualifier": # Skip const, volatile - if node.len() > 1: + if node.len > 1: return node[1].getAtom() else: return @@ -262,10 +262,10 @@ proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if reverse: cnode = cnode.tsNodeParent() else: - if cnode.len() != 0: + if cnode.len != 0: if cnode[0].getName() == "type_qualifier": # Skip const, volatile - if cnode.len() > 1: + if cnode.len > 1: cnode = cnode[1] else: break @@ -285,7 +285,7 @@ proc getDeclarator*(node: TSNode): TSNode = # Return if child is a function or array declarator if node.getName() in ["function_declarator", "array_declarator"]: return node - elif node.len() != 0: + elif node.len != 0: return node[0].getDeclarator() proc firstChildInTree*(node: TSNode, ntype: string): TSNode = @@ -307,7 +307,7 @@ proc anyChildInTree*(node: TSNode, ntype: string): TSNode = for i in 0 ..< cnode.len: let ccnode = cnode[i].anyChildInTree(ntype) - if not ccnode.isNil(): + if not ccnode.isNil: return ccnode if cnode != node: cnode = cnode.tsNodeNextNamedSibling() @@ -326,7 +326,7 @@ proc mostNestedChildInTree*(node: TSNode): TSNode = proc inChildren*(node: TSNode, ntype: string): bool = # Search for node type in immediate children result = false - for i in 0 ..< node.len(): + for i in 0 ..< node.len: if (node[i]).getName() == ntype: result = true break @@ -342,7 +342,7 @@ proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = result.col += 1 proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = - for i in 0 ..< node.len(): + for i in 0 ..< node.len: if node.getName() != "comment": result += 1 @@ -352,11 +352,11 @@ proc getPxName*(node: TSNode, offset: int): string = np = node count = 0 - while not np.isNil() and count < offset: + while not np.isNil and count < offset: np = np.tsNodeParent() count += 1 - if count == offset and not np.isNil(): + if count == offset and not np.isNil: return np.getName() proc printLisp*(gState: State, root: TSNode): string = @@ -366,7 +366,7 @@ proc printLisp*(gState: State, root: TSNode): string = depth = 0 while true: - if not node.isNil() and depth > -1: + if not node.isNil and depth > -1: result &= spaces(depth) let (line, col) = gState.getLineCol(node) @@ -386,7 +386,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= ")\n" nextnode = node.tsNodeNextNamedSibling() - if nextnode.isNil(): + if nextnode.isNil: while true: node = node.tsNodeParent() depth -= 1 @@ -395,7 +395,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= spaces(depth) & ")\n" if node == root: break - if not node.tsNodeNextNamedSibling().isNil(): + if not node.tsNodeNextNamedSibling().isNil: node = node.tsNodeNextNamedSibling() break else: diff --git a/nimterop/lisp.nim b/nimterop/lisp.nim index fb488d8..8287cf5 100644 --- a/nimterop/lisp.nim +++ b/nimterop/lisp.nim @@ -35,7 +35,7 @@ proc readFromTokens(): ref Ast = idx += 2 while gTokens[idx] != ")": var res = readFromTokens() - if not res.isNil(): + if not res.isNil: result.children.add(res) elif gTokens[idx] == ")": doAssert false, "Poor AST " & $(idx: idx) From 25b07a07fff0c15ddae2610016289368fe41f3e2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Apr 2020 22:47:33 -0500 Subject: [PATCH 043/255] ast2 nested array field --- nimterop/ast2.nim | 23 ++++++++++---------- tests/include/tast2.h | 46 ++++++++++++++++++++++++++++++++++++++++ tests/tast2.nim | 49 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 13 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8dba84a..5bb33ae 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -73,7 +73,7 @@ proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: # If not, symbol needs to be skipped - only get here if `name` is blank let # Get cleaned name for symbol, set parent so that cOverride is ignored - name = nimState.getIdentifier(origname, kind, parent = "override") + name = nimState.getIdentifier(origname, kind, parent = "getOverrideOrSkip") override = nimState.getOverride(origname, kind) @@ -492,7 +492,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNo result.add size result.add typ -proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode +proc getTypeArray(nimState: NimState, node: TSNode, tident: PNode, name: string): PNode proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = @@ -637,7 +637,7 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeArray(node[i], node[start], name) + result.add nimState.getTypeArray(node[i], tident, name) result.add newNode(nkEmpty) else: result = nil @@ -958,13 +958,11 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = else: nimState.addTypeObject(node, typeDef = typeDef, istype = true) -proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode = +proc getTypeArray(nimState: NimState, node: TSNode, tident: PNode, name: string): PNode = # Create array type tree + # + # `tident` is type PNode let - # tnode = identifier = type name - (tname, _, info) = nimState.getNameInfo(tnode.getAtom(), nskType, parent = name) - ident = nimState.getIdent(tname, info, exported = false) - # Top-most array declarator adecl = node.firstChildInTree("array_declarator") @@ -990,7 +988,7 @@ proc getTypeArray(nimState: NimState, node, tnode: TSNode, name: string): PNode # ) ncount = innermost[0].getAtom().tsNodeParent().getPtrCount(reverse = true) - result = ident + result = tident var cnode = adecl @@ -1021,8 +1019,9 @@ proc addTypeArray(nimState: NimState, node: TSNode) = let start = getStartAtom(node) - # node[start] = type name - tnode = node[start] + # node[start] = identifier = type name + (tname, _, info) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") + tident = nimState.getIdent(tname, info, exported = false) # Could have multiple types, comma separated for i in start+1 ..< node.len: @@ -1033,7 +1032,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = if not typeDef.isNil: let name = typeDef.getIdentName() - typ = nimState.getTypeArray(node[i], tnode, name) + typ = nimState.getTypeArray(node[i], tident, name) typeDef.add typ diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 136784a..fe78146 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -175,6 +175,29 @@ typedef struct { const char* const* x; } SomeType; +// Nested #137 +typedef struct { + struct NT1 { int f1; } f1; + struct { int f1; } f2; + + struct NT3 { + struct { + int f1; + union NU1 { + float f1; + } f2; + enum { NEV1, NEV2, NEV3 } f3; + } f1; + } f3; + + struct { int f1; } f4; + + union NU2 { int f1; } f5; + union { int f1; } f6; + enum NE1 { NEV4 = 8, NEV5 } f7; + enum { NEV6 = 8 * 8, NEV7 } f8; +} nested; + // DUPLICATES @@ -344,6 +367,29 @@ typedef struct { const char* const* x; } SomeType; +// Nested #137 +typedef struct { + struct NT1 { int f1; } f1; + struct { int f1; } f2; + + struct NT3 { + struct { + int f1; + union NU1 { + float f1; + } f2; + enum { NEV1, NEV2, NEV3 } f3; + } f1; + } f3; + + struct { int f1; } f4; + + union NU2 { int f1; } f5; + union { int f1; } f6; + enum NE1 { NEV4 = 8, NEV5 } f7; + enum { NEV6 = 8 * 8, NEV7 } f8; +} nested; + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index 6d749c6..46101d5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -372,4 +372,51 @@ checkPragmas(AudioCVT, pHeaderBy, istype = false, "SDL_") # Issue #172 assert SomeType is object testFields(SomeType, "x!ptr cstring") -checkPragmas(SomeType, pHeaderImpBy) \ No newline at end of file +checkPragmas(SomeType, pHeaderImpBy) + +# Nested #137 +assert NT1 is object +testFields(NT1, "f1!cint") +checkPragmas(NT1, pHeaderBy, istype = false) + +assert Type_tast2h1 is object +testFields(Type_tast2h1, "f1!cint") +checkPragmas(Type_tast2h1, pHeaderBy, istype = false) + +assert NU1 is object +testFields(NU1, "f1!cfloat") +checkPragmas(NU1, pHeaderBy & @["union"], istype = false) + +assert NEV1 == 0 +assert NEV2 == 1 +assert NEV3 == 2 + +assert Type_tast2h2 is object +testFields(Type_tast2h2, "f1|f2|f3!cint|NU1|Enum_tast2h1") +checkPragmas(Type_tast2h2, pHeaderBy, istype = false) + +assert NT3 is object +testFields(NT3, "f1!Type_tast2h2") +checkPragmas(NT3, pHeaderBy, istype = false) + +assert Type_tast2h3 is object +testFields(Type_tast2h3, "f1!cint") +checkPragmas(Type_tast2h3, pHeaderBy, istype = false) + +assert NU2 is object +testFields(NU2, "f1!cint") +checkPragmas(NU2, pHeaderBy & @["union"], istype = false) + +assert Union_tast2h1 is object +testFields(Union_tast2h1, "f1!cint") +checkPragmas(Union_tast2h1, pHeaderBy & @["union"], istype = false) + +assert NEV4 == 8 +assert NEV5 == 9 + +assert NEV6 == 64 +assert NEV7 == 65 + +assert nested is object +testFields(nested, "f1|f2|f3|f4|f5|f6|f7|f8!NT1|Type_tast2h1|NT3|Type_tast2h3|NU2|Union_tast2h1|NE1|Enum_tast2h2") +checkPragmas(nested, pHeaderImpBy) \ No newline at end of file From 6d08f6ed80ab1195b4d233a82f578d93c8611382 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Apr 2020 12:47:30 -0500 Subject: [PATCH 044/255] Fix enum size to cint --- nimterop/types.nim | 47 ++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/nimterop/types.nim b/nimterop/types.nim index 5209e7a..38f95d1 100644 --- a/nimterop/types.nim +++ b/nimterop/types.nim @@ -22,14 +22,20 @@ type va_list* {.importc, header:"".} = object template enumOp*(op, typ, typout) = - proc op*(x: typ, y: int): typout {.borrow.} - proc op*(x: int, y: typ): typout {.borrow.} + proc op*(x: typ, y: cint): typout {.borrow.} + proc op*(x: cint, y: typ): typout {.borrow.} proc op*(x, y: typ): typout {.borrow.} -template defineEnum*(typ) = - type - typ* = distinct int + proc op*(x: typ, y: int): typout = op(x, y.cint) + proc op*(x: int, y: typ): typout = op(x.cint, y) +template defineEnum*(typ) = + # Create a `distinct cint` type for C enums since Nim enums + # need to be in order and cannot have duplicates. + type + typ* = distinct cint + + # Enum operations allowed enumOp(`+`, typ, typ) enumOp(`-`, typ, typ) enumOp(`*`, typ, typ) @@ -39,29 +45,30 @@ template defineEnum*(typ) = enumOp(`div`, typ, typ) enumOp(`mod`, typ, typ) - proc `shl`*(x: typ, y: int): typ {.borrow.} - proc `shl`*(x: int, y: typ): typ {.borrow.} + # These don't work with `enumOp()` for some reason + proc `shl`*(x: typ, y: cint): typ {.borrow.} + proc `shl`*(x: cint, y: typ): typ {.borrow.} proc `shl`*(x, y: typ): typ {.borrow.} - proc `shr`*(x: typ, y: int): typ {.borrow.} - proc `shr`*(x: int, y: typ): typ {.borrow.} + proc `shr`*(x: typ, y: cint): typ {.borrow.} + proc `shr`*(x: cint, y: typ): typ {.borrow.} proc `shr`*(x, y: typ): typ {.borrow.} - proc `or`*(x: typ, y: int): typ {.borrow.} - proc `or`*(x: int, y: typ): typ {.borrow.} + proc `or`*(x: typ, y: cint): typ {.borrow.} + proc `or`*(x: cint, y: typ): typ {.borrow.} proc `or`*(x, y: typ): typ {.borrow.} - proc `and`*(x: typ, y: int): typ {.borrow.} - proc `and`*(x: int, y: typ): typ {.borrow.} + proc `and`*(x: typ, y: cint): typ {.borrow.} + proc `and`*(x: cint, y: typ): typ {.borrow.} proc `and`*(x, y: typ): typ {.borrow.} - proc `xor`*(x: typ, y: int): typ {.borrow.} - proc `xor`*(x: int, y: typ): typ {.borrow.} + proc `xor`*(x: typ, y: cint): typ {.borrow.} + proc `xor`*(x: cint, y: typ): typ {.borrow.} proc `xor`*(x, y: typ): typ {.borrow.} - proc `/`(x, y: typ): typ = - return (x.float / y.float).int.typ - proc `/`*(x: typ, y: int): typ = `/`(x, y.typ) - proc `/`*(x: int, y: typ): typ = `/`(x.typ, y) + proc `/`*(x, y: typ): typ = + return (x.float / y.float).cint.typ + proc `/`*(x: typ, y: cint): typ = `/`(x, y.typ) + proc `/`*(x: cint, y: typ): typ = `/`(x.typ, y) - proc `$` *(x: typ): string {.borrow.} + proc `$`*(x: typ): string {.borrow.} From 6a9a35db61e1e47652fa667ff557555e87a6edeb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Apr 2020 23:42:22 -0500 Subject: [PATCH 045/255] ast2 varargs support --- nimterop/ast2.nim | 11 +++++++++++ nimterop/getters.nim | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 5bb33ae..554c507 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -686,6 +686,10 @@ proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNo result.add nimState.newFormalParams(name, node, rtyp) result.add nimState.newPragma(node, nimState.gState.convention) + # Add varargs if ... + if node.getVarargs(): + nimState.addPragma(node, result[^1], "varargs") + proc processNode(nimState: NimState, node: TSNode): bool proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object @@ -1582,6 +1586,9 @@ proc addProc(nimState: NimState, node, rnode: TSNode) = # {.impnameC.} shortcut nimState.newPragma(node, nimState.impShort & "C") + # Detect ... and add {.varargs.} + pvarargs = plist.getVarargs() + # Need {.convention.} and {.header.} if applicable if name != origname: if nimState.includeHeader(): @@ -1595,6 +1602,10 @@ proc addProc(nimState: NimState, node, rnode: TSNode) = # {.dynlib.} for DLLs nimState.addPragma(node, prident, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + if pvarargs: + # Add {.varargs.} for ... + nimState.addPragma(node, prident, "varargs") + procDef.add prident procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index aeefbd8..c971bfc 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -288,6 +288,20 @@ proc getDeclarator*(node: TSNode): TSNode = elif node.len != 0: return node[0].getDeclarator() +proc getVarargs*(node: TSNode): bool = + # Detect ... and add {.varargs.} + # + # `node` is the param list + # + # ... is an unnamed node, second last node and ) is last node + let + nlen = node.tsNodeChildCount() + if nlen > 1: + let + nval = node.tsNodeChild(nlen - 2).getName() + if nval == "...": + result = true + proc firstChildInTree*(node: TSNode, ntype: string): TSNode = # Search for node type in tree - first children var From 281d2609122f0697b060c1608fa8df6b70d8edad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Apr 2020 08:39:52 -0500 Subject: [PATCH 046/255] Fix Travis, tests for varargs --- nimterop/getters.nim | 4 ++-- tests/include/tast2.h | 8 ++++---- tests/tast2.nim | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index c971bfc..6cb7651 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -296,9 +296,9 @@ proc getVarargs*(node: TSNode): bool = # ... is an unnamed node, second last node and ) is last node let nlen = node.tsNodeChildCount() - if nlen > 1: + if nlen > 1.uint32: let - nval = node.tsNodeChild(nlen - 2).getName() + nval = node.tsNodeChild(nlen - 2.uint32).getName() if nval == "...": result = true diff --git a/tests/include/tast2.h b/tests/include/tast2.h index fe78146..1c1e935 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -109,7 +109,7 @@ typedef enum VSPresetFormat { // Proc vars void - *(*pcre_malloc)(size_t), + *(*pcre_malloc)(size_t, ...), (*pcre_free)(void *), *(*pcre_stack_malloc)(size_t); @@ -144,7 +144,7 @@ struct ucArrType2 { }; typedef struct fieldfuncfunc { - int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1))); + int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1, ...))); }; int *func2(int f1, int *(*sfunc2)(int f1, int *(*ssfunc2)(int f1))); @@ -311,7 +311,7 @@ typedef enum VSPresetFormat { // Proc vars void - *(*pcre_malloc)(size_t), + *(*pcre_malloc)(size_t, ...), (*pcre_free)(void *), *(*pcre_stack_malloc)(size_t); @@ -336,7 +336,7 @@ void int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); typedef struct fieldfuncfunc { - int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1))); + int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1, ...))); }; int *func2(int f1, int *(*sfunc2)(int f1, int *(*ssfunc2)(int f1))); diff --git a/tests/tast2.nim b/tests/tast2.nim index 46101d5..c49c008 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -302,8 +302,8 @@ assert pfYUV422P8 == pfYUV420P8 + 1 assert pfRGB27 == cmRGB.VSPresetFormat + 11 assert pfCompatYUY2 == pfCompatBGR32 + 1 -assert pcre_malloc is proc(a1: uint): pointer {.cdecl.} -checkPragmas(pcre_malloc, @["importc", "cdecl"] & pHeader) +assert pcre_malloc is proc(a1: uint): pointer {.cdecl, varargs.} +checkPragmas(pcre_malloc, @["importc", "cdecl", "varargs"] & pHeader) assert pcre_free is proc(a1: pointer) {.cdecl.} checkPragmas(pcre_free, @["importc", "cdecl"] & pHeader) @@ -351,7 +351,7 @@ checkPragmas(ucArrType2, pHeaderBy, istype = false) assert fieldfuncfunc is object testFields(fieldfuncfunc, - "func1!proc (f1: cint; sfunc1: proc (f1: cint; ssfunc1: proc (f1: cint): ptr cint {.cdecl.}): ptr cint {.cdecl.}): ptr cint {.cdecl.}") + "func1!proc (f1: cint; sfunc1: proc (f1: cint; ssfunc1: proc (f1: cint): ptr cint {.cdecl, varargs.}): ptr cint {.cdecl.}): ptr cint {.cdecl.}") assert func2 is proc (f1: cint; sfunc2: proc (f1: cint; ssfunc2: proc (f1: cint): ptr cint {.cdecl.}): ptr cint {.cdecl.}): ptr cint {.cdecl.} From e17d6edb5e87f4a74447d3376438dc0d35d95be8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 17 Apr 2020 17:00:11 -0500 Subject: [PATCH 047/255] Update README --- README.md | 168 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 125 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 18f74ce..3f42211 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,13 @@ Detailed documentation [here](https://nimterop.github.io/nimterop/theindex.html) Nimterop is a [Nim](https://nim-lang.org/) package that aims to make C/C++ interop seamless -Nim has one of the best FFI you can find - importing C/C++ is supported out of the box. All you need to provide is type and proc definitions for Nim to interop with C/C++ binaries. Generation of these wrappers is easy for simple libraries but quickly gets out of hand. [c2nim](https://github.com/nim-lang/c2nim) greatly helps here by parsing and converting C/C++ into Nim but is limited due to the complex and constantly evolving C/C++ grammar. [nimgen](https://github.com/genotrance/nimgen) mainly focuses on automating the wrapping process and fills some holes but is again limited to c2nim's capabilities. - -The goal of nimterop is to leverage the [tree-sitter](http://tree-sitter.github.io/tree-sitter/) engine to parse C/C++ code and then convert relevant portions of the AST into Nim definitions. [tree-sitter](https://github.com/tree-sitter) is a Github sponsored project that can parse a variety of languages into an AST which is then leveraged by the [Atom](https://atom.io/) editor for syntax highlighting and code folding. The advantages of this approach are multifold: -- Benefit from the tree-sitter community's investment into language parsing -- Wrap what is recognized in the AST rather than completely failing due to parsing errors - -Most of the wrapping functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how c2nim can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application and other nimgen functionality that helps in automating the wrapping process. There is also support to statically or dynamically link to system installed libraries or downloading and building them with `autoconf` or `cmake` from a Git repo or source archive. +Most of the wrapping functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how `c2nim` can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application and other nimgen functionality that helps in automating the wrapping process. There is also support to statically or dynamically link to system installed libraries or downloading and building them with `autoconf` or `cmake` from a Git repo or source archive. The nimterop wrapping functionality is still limited to C but is constantly expanding. C++ support will be added once most popular C libraries can be wrapped seamlessly. Meanwhile, `c2nim` can also be used in place of `toast` with the `c2nImport()` API call. Nimterop has seen some adoption within the community and the simplicity and success of this approach justifies additional investment of time and effort. Regardless, the goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation. -Also, given tree-sitter can parse a variety of other languages, there might also be value in investigating how to wrap Rust and Go libraries. - -__Installation__ +### Installation Nimterop can be installed via [Nimble](https://github.com/nim-lang/nimble): @@ -34,59 +26,137 @@ nimble develop -y nimble build ``` -This will download and install nimterop in the standard Nimble package location, typically `~/.nimble`. Once installed, it can be imported into any Nim program. Note that the `~/.nimble/bin` directory needs to be added to the `PATH` for nimterop to work. +This will download and install nimterop in the standard Nimble package location, typically `~/.nimble`. Once installed, it can be imported into any Nim program. -__Usage__ +### Usage -Detailed documentation can be found [here](https://nimterop.github.io/nimterop/theindex.html). +Detailed documentation can be found [here](https://nimterop.github.io/nimterop/theindex.html). Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work. -Check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. Please do add your project once you are done so that others can benefit from your work. +#### Build API + +Creating a wrapper has two parts, the first is to setup the C library. This includes downloading it or finding it if already installed, and building it if applicable. The `getHeader()` high-level API provides all of this functionality as a convenience. Following is an example of using the high-level `getHeader()` API to perform all building and linking automatically: -Using the high-level `getHeader` API to perform all building and linking automatically: ```nim import nimterop/[build, cimport] static: - cDebug() + cDebug() # Print wrapper to stdout + +const + baseDir = getProjectCacheDir("testwrapper") # Download library within nimcache getHeader( - "header.h", - giturl = "https://github.com/username/repo", - dlurl = "https://website.org/download/repo-$1.tar.gz", - outdir = "build", - conFlags = "--disable-comp --enable-feature" + "header.h", # The header file to wrap, full path is returned in `headerPath` + giturl = "https://github.com/username/repo", # Git repo URL + dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file + outdir = baseDir, # Where to download/build/search + conFlags = "--disable-comp --enable-feature", # Flags to pass configure script + cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake + altNames = "hdr" # Alterate names of the library binary, full path returned in `headerLPath` ) +# Wrap headerPath as returned from getHeader() and link statically +# or dynamically depending on user input when not defined(headerStatic): - cImport(headerPath, recurse = true, dynlib = "headerLPath") + cImport(headerPath, recurse = true, dynlib = "headerLPath") # Pass dynlib if not static link else: cImport(headerPath, recurse = true) ``` -This allows the user to control how the wrapper works - either pass `-d:headerStd` to search for `header.h` in the standard system path, `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. Further, the `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to use. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL defined. -The `-d:headerStatic` attempts to statically link the library. If it is omitted, the library is dynamically linked instead. +__Download / Search__ -[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library using this high-level API. +The above wrapper is generic and allows the end user to control how it works. + +- If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL if defined. +- If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. + +__Pre build__ + +`getHeader()` provides a `headerPreBuild()` hook that gets called after the library is downloaded but before it is built. This allows for any manipulations of the source files or build scripts before build. [archive](https://github.com/genotrance/nimarchive/blob/master/nimarchive/archive.nim) has such an example. + +The build API also includes various compile time helper procs that aid in file manipulation, Cmake shortcuts, library linking, etc. Refer to [build](https://nimterop.github.io/nimterop/build.html) for more details. + +__Build__ + +Nimterop currently supports `configure` and `cmake` based building of libraries, with `cmake` taking precedence if a project supports both. Nimterop verifies that the tool selected is available and notifies the user if any issues are found. Bash is required on Windows for `configure` and the binary shipped with Git has been tested. + +Flags can be specified to these tools via `getHeader()` or directly via the underlying `configure()` and `cmake()` calls. Once the build scripts are ready, `getHeader()` then calls `make()`. At every step, `getHeader()` checks for the presence of created artifacts and does not redo steps that have been successfully completed. + +__Linking__ + +- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If it is omitted, the dynamic library is returned in `headerLPath`. +- `getHeader()` searches for libraries based on the header name by default: + - `libheader.so` or `libheader.a` on Linux + - `libheader.dylib` on OSX + - `header.dll` or `header.a` on Windows +- If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. + - For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. + - In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. + - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. +- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. + +__User control__ + +The `-d:xxxYYY` Nim define flags have already been described above and can be specified on the command line or in a nim.cfg file. It is also possible to specify them within the wrapper itself using `setDefines()` if required. Further, all defines, regardless of how they are specified, can be generically checked using `isDefined()`. + +If more fine-tuned control is desired over the build process, it is possible to manually control all steps that `getHeader()` performs by directly using the API provided by [build](https://nimterop.github.io/nimterop/build.html). Note also that there is no requirement to use these APIs to setup the library. Any other established mechanisms can be used to do so any limitations imposed by Nimterop are unintentional and feedback is most welcome. + +#### Wrapper API + +Once the C library is setup, the next step is to create wrappers that inform Nim of all the types and functions that are available. Following is a simple example covering the API: -The traditional approach is to manually compile in the code: ```nim import nimterop/cimport static: cDebug() -cDefine("HAS_ABC") -cDefine("HAS_ABC", "DEF") -cIncludeDir("clib/include") -cImport("clib.h") + cDisableCaching() # Regenerate Nim wrapper every time -cCompile("clib/src/*.c") +cDefine("HAS_ABC") # Set #defines for preprocessor and compiler +cDefine("HAS_ABC", "DEF") + +cIncludeDir("clib/include") # Setup any include directories + +cImport("clib.h") # Generate wrappers for header specified + +cCompile("clib/src/*.c") # Compile in any implementation source files ``` -Check out [template.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/template.nim) as a starting point for wrapping a library using the traditional approach. The template can be copied and trimmed down and modified as required. [templite.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/templite.nim) is a shorter version for more experienced users. +Refer to the ```tests``` directory for additional examples on how the library can be used. The [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) is also a good source of examples. -Refer to the ```tests``` directory for examples on how the library can be used. +__Preprocessing__ -The `toast` binary can also be used directly on the CLI: +In order to leverage the preprocessor, certain projects might need `cDefine()` calls to set `#define` values. Simpler library may have documentation that cover this but larger ones will rely on build tools that discover and set values in a `config.h` which is loaded with `#include`. Projects might also require some `cIncludeDir()` calls to specify paths to directories that contain other headers. This might be within the library or refer to another library. + +The wrapper API always runs headers through the C preprocessor before wrapping. Details on why are discussed further down. + +By default, the `$CC` environment variable is used for the compiler path. If not found, `toast` defaults to `gcc`. + +__Wrapping__ + +The `cImport()` call invokes the `toast` binary with appropriate command line flags including any `cDefine()` and `cInclude()` parameters configured. The output of `toast` is then pulled into the module as Nim code and printed if `cDebug()` is specified. This allows for an end user to simply import the wrapper into their code and access the library API as Nim types and procs. Output is cached to save time on subsequent runs. It is also possible to just redirect the output to a file and import that instead if preferred. + +The `recurse` flag can be set to enable the recursion capability which runs through all #include files in the header. If the library needs to be dyamically linked using Nim's `dynlib` pragma, the `dynlib = "constName"` attribute can be set to generate wrappers that load the DLL automatically. Without `dynlib`, static link is assumed so it is the user's responsibility to link the library. + +There may be cases where the wrapper generated by `toast` for certain types or procs is not preferred, or may be skipped or altogether wrong due to limitations or bugs. In these instances, the `cOverride()` macro can be used to define consts, types or procs to use in place of the wrapper generated output. `cImport()` will forward this information to `toast` and the values will be inserted in context in the generated wrapper. This allows wrapper authors to work around tool limitations or to improve the wrapper output - say change `ptr X` to `var X` or to create more Nim friendly types or proc signatures. + +Several C libraries also use leading and/or trailing `_` in identifiers and since Nim does not allow this, the `cPlugin()` macro can be used to modify such symbols or `cSkipSymbol()` them altogether. Instead of a full `cPlugin()` section, it might also be preferred to set `flags = "-E_ -F_"` to the `cImport()` call to trim out such characters. These features can also be used to remove common prefixes like `SDL_` to generate a cleaner wrapper. `cPlugin()` is real Nim code though so anything Nim allows is fair game. Note that `cPlugin()` overrides any `-E -F` flags. Also, behind the scenes, `cOverride()` is communicated to `toast` via `cPlugin()`. + +If the same `cPlugin()` is needed in multiple wrapper files, the code can be moved into a standalone file and be used with the `cPluginPath()` call. + +Lastly, `c2nImport()` provides access to calling `c2nim` from the wrapper instead of `toast`. Note that `c2nImport()` does not use any of the above described features like `cPlugin()` and needs to be controlled with the `flags` param. + +__Compiling source__ + +The job of building and compiling the underlying C library is best left to the build mechanism selected by the library author so using `getHeader()` is recommended. For simpler projects with a few `.c` files though, `cCompile()` should be more than enough. It is not recommended for larger projects which heavily rely on functionality offered by build tools. Recreating reliable logic in Nim can be tedious and one can expect minimal support from that author if their tested build mechanism is not used. + +#### Command line API + +The `toast` binary can also be used directly on the CLI, similar to `c2nim`. The `cPlugin()` interface + +Note: unlike the wrapper API, the `-p | --preprocess` flag is not enabled by default but is *highly* recommended. ``` > toast -h @@ -96,17 +166,17 @@ Options: -h, --help print this cligen-erated help --help-syntax advanced: prepend,plurals,.. -k, --check bool false check generated wrapper with compiler - -C=, --convention= string "cdecl" calling convention for wrapped procs - default: cdecl + -C=, --convention= string "cdecl" calling convention for wrapped procs -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" import symbols from library in specified Nim string -f=, --feature= Features {} flags to enable experimental features -H, --includeHeader bool false add {.header.} pragma to wrapper -I=, --includeDirs= strings {} include directory to pass to preprocessor - -m=, --mode= string "cpp" language parser: c or cpp - --nim= string "nim" use a particular Nim executable - default: $PATH/nim + -m=, --mode= string "" language parser: c or cpp + --nim= string "nim" use a particular Nim executable -c, --nocomments bool false exclude top-level comments from output - -o=, --output= string "" file to output content - default: stdout + -o=, --output= string "" file to output content -a, --past bool false print AST output -g, --pgrammar bool false print grammar --pluginSourcePath= string "" nim file to build and load as a plugin @@ -119,15 +189,27 @@ Options: -O=, --symOverride= strings {} skip generating specified symbols ``` -__Implementation Details__ +### Why nimterop -In order to use the tree-sitter C library, it has to be compiled into a separate binary called `toast` (to AST) since the Nim VM doesn't yet support FFI. `toast` takes a C/C++ file and runs it through the tree-sitter API which returns an AST data structure. This can then be printed out to stdout in a Lisp S-Expression format or the relevant Nim wrapper output. This content can be saved to a `.nim` file and imported if so desired. +Nim has one of the best FFI you can find - importing C/C++ is supported out of the box. All you need to provide is type and proc definitions for Nim to interop with C/C++ binaries. Generation of these wrappers is easy for simple libraries but can quickly get out of hand. [c2nim](https://github.com/nim-lang/c2nim) greatly helps here by parsing and converting C/C++ into Nim but is limited due to the complex and constantly evolving C/C++ grammar. [nimgen](https://github.com/genotrance/nimgen) mainly focused on automating the wrapping process with `c2nim` and filled some holes but is again limited to `c2nim` capabilities. -Alternatively, the `cImport()` macro allows easier creation of wrappers in code. It runs `toast` on the specified header file and injects the generated wrapper content into the application at compile time. A few other helper procs are provided to influence this process. Output is cached to save time on subsequent runs. +The goal of nimterop is to leverage the [tree-sitter](http://tree-sitter.github.io/tree-sitter/) engine to parse C/C++ code and then convert relevant portions of the AST into Nim definitions. [tree-sitter](https://github.com/tree-sitter) is a Github sponsored project that can parse a variety of languages into an AST which is then leveraged by the [Atom](https://atom.io/) editor for syntax highlighting and code folding. The advantages of this approach are multifold: +- Benefit from the tree-sitter community's ongoing investment into language parsing +- Wrap what is recognized in the AST rather than completely failing due to parsing errors -`toast` can also be used to run the header through the preprocessor which cleans up the code considerably. Along with the recursion capability which runs through all #include files, one large simpler header file can be created which can then be processed with `toast` or even `c2nim` if so desired. By default, the `$CC` environment variable is used. If not found, `toast` defaults to `gcc`. +The tree-sitter library is limited though - it may fail on some advanced language constructs but is designed to handle them gracefully since it is expected to have bad code while actively typing in an editor. When an error is detected, tree-sitter includes an ERROR node at that location in the AST. At this time, `cImport()` will complain and continue if it encounters any errors. Depending on how severe the errors are, compilation may succeed or fail. Glaring issues will be communicated to the tree-sitter team but their goals may not always align with those of this project. -The tree-sitter library is limited as well - it may fail on some advanced language constructs but is designed to handle them gracefully since it is expected to have bad code while actively typing in an editor. When an error is detected, tree-sitter includes an ERROR node at that location in the AST. At this time, `cImport()` will complain and continue if it encounters any errors. Depending on how severe the errors are, compilation may succeed or fail. Glaring issues will be communicated to the tree-sitter team but their goals may not always align with those of this project. +It is debatable whether a syntax highlighting engine like `tree-sitter` is the most reliable method to convert C code into AST. However, it is lightweight, cross-platform with no dependencies and handles error conditions gracefully. It has produced usable wrappers for C libraries though things could get murky when considering C++ but that will be a topic for another day. Nimterop relies heavily on the preprocessor, as discussed next, so having an engine which can run anywhere has been worth the compromise. Only time will tell though. + +__Preprocessing__ + +The wrapper API always runs headers through the C preprocessor before wrapping, unlike the command line interface where the `-p | --preprocess` flag is not set by default but *highly* recommended. This is because almost all platform, compiler and package discovery is handled by build tools like `configure` and `cmake` which then use preprocessor `#define` values to tweak what C code is applicable for that platform. While parsing preprocessor macros is possible in tools like `toast`, given how dependent the `#ifdef` branches are on values provided by these and many other build tools, preprocessing seems is best left to them than attempting to self-discover or intercept that information. + +Nimterop is still able to wrap most relevant `#define` like numbers and strings thanks to `gcc -E` providing the sufficient detail in its output. Many C libraries also use `#define` templates for some of their user facing API and providing that functionality in Nim is on the Nimterop roadmap. + +The con of this approach of delegating to the preprocessor is that the Nim wrapper generated by Nimterop is no longer portable despite being Nim code. A wrapper rendered on Linux might not work on Windows since some APIs may not be available or inappropriate, integer sizes might be wrong, types could be missing and many other possible issues. But none of this is easily or accurately known at the Nim level since it would require input from the build tools which already work well with the preprocessor or have to be completely reimplemented within Nim. Neither approach that bypasses such build tools would be supported by the library author. + +This is part of the reason why Nimterop provides a wrapper API so that the generation of wrappers is Nim code that can be rendered as part of the build process on the target platform. It helps to think of Nimterop as a build time tool like `cmake` that renders artifacts on the target rather than a tool whose generated artifacts should be checked into source control. Regardless, both the wrapper API and the `toast` command line still allow saving the wrapper output to a file to be stored in source control since it might work well enough for many projects. __Credits__ From 485c2f1ab2436094c28eabd97a3a1c02abcddf82 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 19 Apr 2020 00:50:14 -0500 Subject: [PATCH 048/255] ast2 static inline support --- nimterop/ast2.nim | 6 ++++++ nimterop/getters.nim | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 554c507..92d9fc8 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1657,6 +1657,12 @@ proc processNode(nimState: NimState, node: TSNode): bool = nimState.addEnum(node) of "declaration": nimState.addDecl(node) + of "function_definition": + # Handle static inline + let + start = getStartAtom(node) + if node[start+1].getName() == "function_declarator": + nimState.addProc(node[start+1], node[start]) else: # Unknown result = false diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 6cb7651..502ec89 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -568,8 +568,7 @@ proc getPreprocessor*(gState: State, fullpath: string): string = rdata.add line return rdata.join("\n"). replace("__restrict", ""). - replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1"). - removeStatic() + replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1") converter toString*(kind: Kind): string = return case kind: From ab0439e60eab40a4e21d95261a8699690984b6a2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 20 Apr 2020 12:28:14 -0500 Subject: [PATCH 049/255] ast2 static inline with -H, fix #188, --replace support --- README.md | 1 + nimterop/all.nim | 4 +--- nimterop/ast2.nim | 22 +++++++++++++++++----- nimterop/getters.nim | 9 ++++++++- nimterop/globals.nim | 2 ++ nimterop/toast.nim | 16 ++++++++++++++-- tests/include/tast2.h | 8 ++++++++ tests/tast2.nim | 15 ++++++++++----- 8 files changed, 61 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3f42211..0036187 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ Options: -E=, --prefix= strings {} strip prefix from identifiers -p, --preprocess bool false run preprocessor on header -r, --recurse bool false process #include files + -G=, --replace= strings {} replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers -O=, --symOverride= strings {} skip generating specified symbols diff --git a/nimterop/all.nim b/nimterop/all.nim index fef1bdd..6be6d11 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -1,7 +1,5 @@ ##[ -Module that should import everything so that `nim doc --project nimtero/all` runs docs on everything. +The following modules are available to users of Nimterop. ]## -# TODO: make sure it does import everything. - import "."/[docs, cimport, build, types, plugin] diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 92d9fc8..23bf774 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1636,6 +1636,22 @@ proc addDecl(nimState: NimState, node: TSNode) = # Regular var discard +proc addDef(nimState: NimState, node: TSNode) = + # Wrap static inline definition if {.header.} mode is specified + # + # Without {.header.} the definition will not be available to the C compiler + # and will fail at link time + decho("addDef()") + nimState.printDebug(node) + let + start = getStartAtom(node) + if node[start+1].getName() == "function_declarator": + if nimState.includeHeader(): + nimState.addProc(node[start+1], node[start]) + else: + necho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % + nimState.getNodeVal(node[start+1].getAtom()) + proc processNode(nimState: NimState, node: TSNode): bool = result = true @@ -1658,11 +1674,7 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "declaration": nimState.addDecl(node) of "function_definition": - # Handle static inline - let - start = getStartAtom(node) - if node[start+1].getName() == "function_declarator": - nimState.addProc(node[start+1], node[start]) + nimState.addDef(node) else: # Unknown result = false diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 502ec89..798ad24 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -137,6 +137,13 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent=" if result.endsWith(str): result = result[0 .. ^(str.len+1)] + # --replace from CLI if specified + for name, value in nimState.gState.replace.pairs: + if name.len > 1 and name[0] == '@': + result = result.replace(re(name[1 .. ^1]), value) + else: + result = result.replace(name, value) + checkIdentifier(result, $kind, parent, name) if result in gReserved or (result == "object" and kind != nskType): @@ -738,6 +745,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath() + result = path.expandFilename().expandSymlink().normalizedPath() except: result = path diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 42f7cb7..47157d8 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -61,6 +61,8 @@ type code*, convention*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string + replace*: OrderedTableRef[string, string] + feature*: seq[Feature] onSymbol*, onSymbolOverride*: OnSymbol diff --git a/nimterop/toast.nim b/nimterop/toast.nim index c3d6a3d..3f886e1 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -1,4 +1,4 @@ -import os, osproc, strformat, strutils, times +import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] @@ -67,6 +67,7 @@ proc main( prefix: seq[string] = @[], preprocess = false, recurse = false, + replace: seq[string] = @[], stub = false, suffix: seq[string] = @[], symOverride: seq[string] = @[], @@ -91,6 +92,7 @@ proc main( prefix: prefix, preprocess: preprocess, recurse: recurse, + replace: newOrderedTable[string, string](), suffix: suffix, symOverride: symOverride ) @@ -104,6 +106,14 @@ proc main( gState.prefix = gState.prefix.getSplitComma() gState.suffix = gState.suffix.getSplitComma() + # Replace => Table + for i in replace.getSplitComma(): + let + nv = i.split("=", maxsplit = 1) + name = nv[0] + value = if nv.len == 2: nv[1] else: "" + gState.replace[name] = value + if pluginSourcePath.nBl: gState.loadPlugin(pluginSourcePath) @@ -205,10 +215,11 @@ when isMainModule: "pgrammar": "print grammar", "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", + "prefix": "strip prefix from identifiers", "preprocess": "run preprocessor on header", "recurse": "process #include files", + "replace": "replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex", "source" : "C/C++ source/header", - "prefix": "strip prefix from identifiers", "stub": "stub out undefined type references as objects", "suffix": "strip suffix from identifiers", "symOverride": "skip generating specified symbols" @@ -229,6 +240,7 @@ when isMainModule: "prefix": 'E', "preprocess": 'p', "recurse": 'r', + "replace": 'G', "stub": 's', "suffix": 'F', "symOverride": 'O' diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 1c1e935..bdf8823 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -198,6 +198,10 @@ typedef struct { enum { NEV6 = 8 * 8, NEV7 } f8; } nested; +static inline int sitest1(int f1) { + return f1 * 2; +} + // DUPLICATES @@ -390,6 +394,10 @@ typedef struct { enum { NEV6 = 8 * 8, NEV7 } f8; } nested; +static inline int sitest1(int f1) { + return f1 * 2; +} + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index c49c008..e9d9ced 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -32,7 +32,7 @@ cOverride: type A1* = A0 -cImport(path, flags="-f:ast2 -ENK_,SDL_" & flags) +cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE" & flags) proc getPragmas(n: NimNode): HashSet[string] = # Find all pragmas in AST, return as "name" or "name:value" in set @@ -355,9 +355,10 @@ testFields(fieldfuncfunc, assert func2 is proc (f1: cint; sfunc2: proc (f1: cint; ssfunc2: proc (f1: cint): ptr cint {.cdecl.}): ptr cint {.cdecl.}): ptr cint {.cdecl.} -assert BASS_DEVICEINFO is object -testFields(BASS_DEVICEINFO, "name|driver|flags!cstring|cstring|cint") -checkPragmas(BASS_DEVICEINFO, pHeaderImpBy) +# Test --replace VICE=SLICE +assert BASS_DESLICEINFO is object +testFields(BASS_DESLICEINFO, "name|driver|flags!cstring|cstring|cint") +checkPragmas(BASS_DESLICEINFO, pHeaderImpBy) # Issue #183 assert GPU_Target is object @@ -419,4 +420,8 @@ assert NEV7 == 65 assert nested is object testFields(nested, "f1|f2|f3|f4|f5|f6|f7|f8!NT1|Type_tast2h1|NT3|Type_tast2h3|NU2|Union_tast2h1|NE1|Enum_tast2h2") -checkPragmas(nested, pHeaderImpBy) \ No newline at end of file +checkPragmas(nested, pHeaderImpBy) + +when defined(HEADER): + assert sitest1(5) == 10 + assert sitest1(10) == 20 \ No newline at end of file From e7bf0f61ad3629db7806968526e0f2014bb3a045 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 20 Apr 2020 13:24:57 -0500 Subject: [PATCH 050/255] ast2 fix replace test --- tests/tast2.nim | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/tast2.nim b/tests/tast2.nim index e9d9ced..e13c4ac 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -58,7 +58,7 @@ proc getRecList(n: NimNode): NimNode = return rl macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] = true, - prefix: static[string] = ""): untyped = + origname: static[string] = ""): untyped = # Verify that type has expected pragmas defined # `istype` is true when typedef X var @@ -68,9 +68,12 @@ macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] when defined(HEADER): if not istype: if "union" in exprag: - exprag.incl "importc:union " & $prefix & $t + exprag.incl "importc:union " & $t else: - exprag.incl "importc:struct " & $prefix & $t + exprag.incl "importc:struct " & $t + elif origname.len != 0: + exprag.incl "importc:" & $origname + doAssert symmetricDifference(prag, exprag).len == 0, "\nWrong number of pragmas in " & $t & "\n" & $prag & " vs " & $exprag @@ -358,7 +361,7 @@ assert func2 is proc (f1: cint; sfunc2: proc (f1: cint; ssfunc2: proc (f1: cint) # Test --replace VICE=SLICE assert BASS_DESLICEINFO is object testFields(BASS_DESLICEINFO, "name|driver|flags!cstring|cstring|cint") -checkPragmas(BASS_DESLICEINFO, pHeaderImpBy) +checkPragmas(BASS_DESLICEINFO, pHeaderBy, origname = "BASS_DEVICEINFO") # Issue #183 assert GPU_Target is object @@ -368,7 +371,7 @@ checkPragmas(GPU_Target, pHeaderBy, istype = false) # Issue #185 assert AudioCVT is object testFields(AudioCVT, "needed!cint") -checkPragmas(AudioCVT, pHeaderBy, istype = false, "SDL_") +checkPragmas(AudioCVT, pHeaderBy, origname = "struct SDL_AudioCVT") # Issue #172 assert SomeType is object From 40c8e757aa5188618b7bc8abf3e913b08269dd28 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 20 Apr 2020 19:00:55 -0500 Subject: [PATCH 051/255] Remove expandSymlink --- nimterop/getters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 798ad24..654c7f3 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -745,6 +745,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandFilename().expandSymlink().normalizedPath() + result = path.expandFilename().normalizedPath() except: result = path From 82ca3bd37b1587e3c39708e51ed44375b322c111 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Tue, 21 Apr 2020 06:41:16 +0200 Subject: [PATCH 052/255] Some small compatibility fixes for FreeBSD (#190) --- nimterop/build.nim | 6 +++--- nimterop/cimport.nim | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index cc6dc22..a025dc0 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -358,7 +358,7 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" elif defined(linux): "find $3 $1 -regextype egrep -regex $2" - elif defined(osx): + elif defined(osx) or defined(FreeBSD): "find -E $3 $1 -regex $2" recursive = "" @@ -754,7 +754,7 @@ proc getNumProcs(): string = getEnv("NUMBER_OF_PROCESSORS").strip() elif defined(linux): execAction("nproc").output.strip() - elif defined(macosx): + elif defined(macosx) or defined(FreeBSD): execAction("sysctl -n hw.ncpu").output.strip() else: "1" @@ -830,7 +830,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin proc getDynlibExt(): string = when defined(windows): result = ".dll" - elif defined(linux): + elif defined(linux) or defined(FreeBSD): result = ".so[0-9.]*" elif defined(macosx): result = ".dylib[0-9.]*" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 382c034..52d1892 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -82,7 +82,7 @@ proc getFileDate(fullpath: string): string = &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" elif defined(Linux): &"stat -c %y {fullpath.sanitizePath}" - elif defined(OSX): + elif defined(OSX) or defined(FreeBSD): &"stat -f %m {fullpath.sanitizePath}" (result, ret) = execAction(cmd) From 01bc01a30c25ccd6bc3e344adc1f6be38500b6b0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 21 Apr 2020 13:39:20 -0500 Subject: [PATCH 053/255] Handle qualifiers after type --- nimterop/ast2.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 23bf774..7c32a21 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -529,9 +529,10 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So # struct ABC { int w, h; }; # # This is not applicable for procs. - let + var start = getStartAtom(node) + let # node[start] - param type (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) @@ -544,6 +545,10 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So tident = nimState.getIdent(tname, tinfo, exported = false) + # Skip qualifiers after type + while start < node.len - 1 and node[start+1].getName() == "type_qualifier": + start += 1 + if start == node.len - 1: # Only for proc with no named param - create a param name based on offset # From 183eabff83aa7c2cda6fcd41427ad95b3d4fd7ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 21 Apr 2020 14:53:26 -0500 Subject: [PATCH 054/255] Get rid of nimState --- nimterop/ast.nim | 156 ++++++------ nimterop/ast2.nim | 562 +++++++++++++++++++++---------------------- nimterop/getters.nim | 121 +++++----- nimterop/globals.nim | 17 +- nimterop/grammar.nim | 350 +++++++++++++-------------- 5 files changed, 595 insertions(+), 611 deletions(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 06b0504..6b21bc7 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -4,27 +4,27 @@ import regex import "."/[getters, globals, treesitter/api] -proc getHeaderPragma*(nimState: NimState): string = +proc getHeaderPragma*(gState: State): string = result = - if nimState.includeHeader(): - &", header: {nimState.currentHeader}" + if gState.isIncludeHeader(): + &", header: {gState.currentHeader}" else: "" -proc getDynlib*(nimState: NimState): string = +proc getDynlib*(gState: State): string = result = - if nimState.gState.dynlib.nBl: - &", dynlib: {nimState.gState.dynlib}" + if gState.dynlib.nBl: + &", dynlib: {gState.dynlib}" else: "" -proc getImportC*(nimState: NimState, origName, nimName: string): string = +proc getImportC*(gState: State, origName, nimName: string): string = if nimName != origName: - result = &"importc: \"{origName}\"{nimState.getHeaderPragma()}" + result = &"importc: \"{origName}\"{gState.getHeaderPragma()}" else: - result = nimState.impShort + result = gState.impShort -proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = +proc getPragma*(gState: State, pragmas: varargs[string]): string = result = "" for pragma in pragmas.items(): if pragma.nBl: @@ -32,15 +32,15 @@ proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = if result.nBl: result = " {." & result[0 .. ^3] & ".}" - result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C") + result = result.replace(gState.impShort & ", cdecl", gState.impShort & "C") let - dy = nimState.getDynlib() + dy = gState.getDynlib() if ", cdecl" in result and dy.nBl: result = result.replace(".}", dy & ".}") -proc saveNodeData(node: TSNode, nimState: NimState): bool = +proc saveNodeData(node: TSNode, gState: State): bool = let name = $node.tsNodeType() # Atoms are nodes whose values are to be saved @@ -52,7 +52,7 @@ proc saveNodeData(node: TSNode, nimState: NimState): bool = ppppname = node.getPxName(4) var - val = nimState.getNodeVal(node) + val = gState.getNodeVal(node) # Skip since value already obtained from parent atom if name == "primitive_type" and pname == "sized_type_specifier": @@ -64,7 +64,7 @@ proc saveNodeData(node: TSNode, nimState: NimState): bool = # Add reference point in saved data for bitfield_clause if name in ["number_literal"] and pname == "bitfield_clause": - nimState.data.add(("bitfield_clause", val)) + gState.data.add(("bitfield_clause", val)) return true # Process value as a type @@ -74,52 +74,52 @@ proc saveNodeData(node: TSNode, nimState: NimState): bool = if node.tsNodePrevNamedSibling().tsNodeIsNull(): if pname == "pointer_declarator": if ppname notin ["function_declarator", "array_declarator"]: - nimState.data.add(("pointer_declarator", "")) + gState.data.add(("pointer_declarator", "")) elif ppname == "array_declarator": - nimState.data.add(("array_pointer_declarator", "")) + gState.data.add(("array_pointer_declarator", "")) # Double pointer if ppname == "pointer_declarator": - nimState.data.add(("pointer_declarator", "")) + gState.data.add(("pointer_declarator", "")) elif pname in ["function_declarator", "array_declarator"]: if ppname == "pointer_declarator": - nimState.data.add(("pointer_declarator", "")) + gState.data.add(("pointer_declarator", "")) if pppname == "pointer_declarator": - nimState.data.add(("pointer_declarator", "")) + gState.data.add(("pointer_declarator", "")) - nimState.data.add((name, val)) + gState.data.add((name, val)) if pname == "pointer_declarator" and ppname == "function_declarator": if name == "field_identifier": if pppname == "pointer_declarator": - nimState.data.insert(("pointer_declarator", ""), nimState.data.len-1) + gState.data.insert(("pointer_declarator", ""), gState.data.len-1) if ppppname == "pointer_declarator": - nimState.data.insert(("pointer_declarator", ""), nimState.data.len-1) - nimState.data.add(("function_declarator", "")) + gState.data.insert(("pointer_declarator", ""), gState.data.len-1) + gState.data.add(("function_declarator", "")) elif name == "identifier": - nimState.data.add(("pointer_declarator", "")) + gState.data.add(("pointer_declarator", "")) # Save node value for a top-level expression elif name in gExpressions and name != "escape_sequence": if $node.tsNodeParent.tsNodeType() notin gExpressions: - nimState.data.add((name, nimState.getNodeVal(node))) + gState.data.add((name, gState.getNodeVal(node))) elif name in ["abstract_pointer_declarator", "enumerator", "field_declaration", "function_declarator"]: - nimState.data.add((name.replace("abstract_", ""), "")) + gState.data.add((name.replace("abstract_", ""), "")) return true -proc searchAstForNode(ast: ref Ast, node: TSNode, nimState: NimState): bool = +proc searchAstForNode(ast: ref Ast, node: TSNode, gState: State): bool = let childNames = node.getTSNodeNamedChildNames().join() if ast.isNil: return - if nimState.gState.debug: - nimState.nodeBranch.add $node.tsNodeType() - necho "#" & spaces(nimState.nodeBranch.len * 2) & nimState.nodeBranch[^1] + if gState.debug: + gState.nodeBranch.add $node.tsNodeType() + gecho "#" & spaces(gState.nodeBranch.len * 2) & gState.nodeBranch[^1] if ast.children.nBl: if childNames.contains(ast.regex) or @@ -137,26 +137,26 @@ proc searchAstForNode(ast: ref Ast, node: TSNode, nimState: NimState): bool = else: ast - if not searchAstForNode(astChild, nodeChild, nimState): + if not searchAstForNode(astChild, nodeChild, gState): flag = false break if flag: - result = node.saveNodeData(nimState) + result = node.saveNodeData(gState) else: - result = node.saveNodeData(nimState) + result = node.saveNodeData(gState) else: - if nimState.gState.debug: - necho "#" & spaces(nimState.nodeBranch.len * 2) & &" {ast.getRegexForAstChildren()} !=~ {childNames}" + if gState.debug: + gecho "#" & spaces(gState.nodeBranch.len * 2) & &" {ast.getRegexForAstChildren()} !=~ {childNames}" elif node.getTSNodeNamedChildCountSansComments() == 0: - result = node.saveNodeData(nimState) + result = node.saveNodeData(gState) - if nimState.gState.debug: - discard nimState.nodeBranch.pop() - if nimstate.nodeBranch.Bl: - necho "" + if gState.debug: + discard gState.nodeBranch.pop() + if gState.nodeBranch.Bl: + gecho "" -proc searchAst(root: TSNode, astTable: AstTable, nimState: NimState) = +proc searchAst(root: TSNode, astTable: AstTable, gState: State) = var node = root nextnode: TSNode @@ -168,14 +168,14 @@ proc searchAst(root: TSNode, astTable: AstTable, nimState: NimState) = name = $node.tsNodeType() if name in astTable: for ast in astTable[name]: - if nimState.gState.debug: - necho "\n# " & nimState.getNodeVal(node).replace("\n", "\n# ") & "\n" - if searchAstForNode(ast, node, nimState): - ast.tonim(ast, node, nimState) - if nimState.gState.debug: - nimState.debugStr &= "\n# " & nimState.data.join("\n# ") & "\n" + if gState.debug: + gecho "\n# " & gState.getNodeVal(node).replace("\n", "\n# ") & "\n" + if searchAstForNode(ast, node, gState): + ast.tonim(ast, node, gState) + if gState.debug: + gState.debugStr &= "\n# " & gState.data.join("\n# ") & "\n" break - nimState.data = @[] + gState.data = @[] else: break @@ -204,48 +204,46 @@ proc searchAst(root: TSNode, astTable: AstTable, nimState: NimState) = proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = var - nimState = new(NimState) fp = fullpath.replace("\\", "/") - nimState.identifiers = newTable[string, string]() + gState.identifiers = newTable[string, string]() - nimState.gState = gState - nimState.currentHeader = getCurrentHeader(fullpath) - nimState.impShort = nimState.currentHeader.replace("header", "imp") - nimState.sourceFile = fullpath + gState.currentHeader = getCurrentHeader(fullpath) + gState.impShort = gState.currentHeader.replace("header", "imp") + gState.sourceFile = fullpath - if nimState.includeHeader(): - nimState.constStr &= &"\n {nimState.currentHeader} {{.used.}} = \"{fp}\"" + if gState.isIncludeHeader(): + gState.constStr &= &"\n {gState.currentHeader} {{.used.}} = \"{fp}\"" - root.searchAst(astTable, nimState) + root.searchAst(astTable, gState) - if nimState.enumStr.nBl: - necho &"{nimState.enumStr}\n" + if gState.enumStr.nBl: + gecho &"{gState.enumStr}\n" - nimState.constStr = nimState.getOverrideFinal(nskConst) & nimState.constStr - if nimState.constStr.nBl: - necho &"const{nimState.constStr}\n" + gState.constStr = gState.getOverrideFinal(nskConst) & gState.constStr + if gState.constStr.nBl: + gecho &"const{gState.constStr}\n" - necho &""" -{{.pragma: {nimState.impShort}, importc{nimState.getHeaderPragma()}.}} -{{.pragma: {nimState.impShort}C, {nimState.impShort}, cdecl{nimState.getDynlib()}.}} + gecho &""" +{{.pragma: {gState.impShort}, importc{gState.getHeaderPragma()}.}} +{{.pragma: {gState.impShort}C, {gState.impShort}, cdecl{gState.getDynlib()}.}} """ - nimState.typeStr = nimState.getOverrideFinal(nskType) & nimState.typeStr - if nimState.typeStr.nBl: - necho &"type{nimState.typeStr}\n" + gState.typeStr = gState.getOverrideFinal(nskType) & gState.typeStr + if gState.typeStr.nBl: + gecho &"type{gState.typeStr}\n" - nimState.procStr = nimState.getOverrideFinal(nskProc) & nimState.procStr - if nimState.procStr.nBl: - necho &"{nimState.procStr}\n" + gState.procStr = gState.getOverrideFinal(nskProc) & gState.procStr + if gState.procStr.nBl: + gecho &"{gState.procStr}\n" - if nimState.gState.debug: - if nimState.debugStr.nBl: - necho nimState.debugStr + if gState.debug: + if gState.debugStr.nBl: + gecho gState.debugStr - if nimState.skipStr.nBl: + if gState.skipStr.nBl: let - hash = nimState.skipStr.hash().abs() + hash = gState.skipStr.hash().abs() sname = getTempDir() / &"nimterop_{$hash}.h" - necho &"# Writing skipped definitions to {sname}\n" - writeFile(sname, nimState.skipStr) + gecho &"# Writing skipped definitions to {sname}\n" + writeFile(sname, gState.skipStr) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 7c32a21..9e8f582 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -22,17 +22,17 @@ proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) if msg < warnMin: raise newException(Exception, msgKindToString(msg)) -proc parseString(nimState: NimState, str: string): PNode = +proc parseString(gState: State, str: string): PNode = # Parse a string into Nim AST - use custom error handler that raises # an exception rather than exiting on failure try: result = parseString( - str, nimState.identCache, nimState.config, errorHandler = handleError + str, gState.identCache, gState.config, errorHandler = handleError ) except: decho getCurrentExceptionMsg() -proc getLit*(nimState: NimState, str: string, expression = false): PNode = +proc getLit*(gState: State, str: string, expression = false): PNode = # Used to convert #define literals into const and expressions # in array sizes # @@ -47,7 +47,7 @@ proc getLit*(nimState: NimState, str: string, expression = false): PNode = result = newFloatNode(nkFloatLit, parseFloat(str)) elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - result = nimState.parseString(str) + result = gState.parseString(str) elif str.contains(re"^'[[:ascii:]]'$"): # char result = newNode(nkCharLit) @@ -59,23 +59,23 @@ proc getLit*(nimState: NimState, str: string, expression = false): PNode = else: let str = - if expression: nimState.getNimExpression(str) + if expression: gState.getNimExpression(str) else: str - result = nimState.parseString(str) + result = gState.parseString(str) if result.isNil: result = newNode(nkNilLit) -proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode = +proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined # and use that if present # # If not, symbol needs to be skipped - only get here if `name` is blank let # Get cleaned name for symbol, set parent so that cOverride is ignored - name = nimState.getIdentifier(origname, kind, parent = "getOverrideOrSkip") + name = gState.getIdentifier(origname, kind, parent = "getOverrideOrSkip") - override = nimState.getOverride(origname, kind) + override = gState.getOverride(origname, kind) var skind = getKeyword(kind) & " " @@ -83,25 +83,25 @@ proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: if kind == nskProc: skind = "" let - pnode = nimState.parseString(skind & override.replace(origname, name)) + pnode = gState.parseString(skind & override.replace(origname, name)) if not pnode.isNil: result = pnode[0][0] else: - necho &"\n# $1'{origname}' skipped" % skind - if nimState.gState.debug: - nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" + gecho &"\n# $1'{origname}' skipped" % skind + if gState.debug: + gState.skipStr &= &"\n{gState.getNodeVal(node)}" -proc addOverrideFinal(nimState: NimState, kind: NimSymKind) = +proc addOverrideFinal(gState: State, kind: NimSymKind) = # Add all unused cOverride symbols for `kind` to AST var - syms = nimState.getOverrideFinal(kind) + syms = gState.getOverrideFinal(kind) skind = getKeyword(kind) & "\n" if kind == nskProc: skind = "" if syms.nBl: var - nsyms = nimState.parseString(skind & syms) + nsyms = gState.parseString(skind & syms) if not nsyms.isNil: let list = @@ -111,20 +111,20 @@ proc addOverrideFinal(nimState: NimState, kind: NimSymKind) = nsyms[0].sons case kind of nskConst: - nimState.constSection.sons.insert(list, 0) + gState.constSection.sons.insert(list, 0) of nskType: - nimState.typeSection.sons.insert(list, 0) + gState.typeSection.sons.insert(list, 0) of nskProc: - nimState.procSection.sons.insert(list, 0) + gState.procSection.sons.insert(list, 0) else: discard -proc addAllOverrideFinal(nimState: NimState) = +proc addAllOverrideFinal(gState: State) = # Add all unused cOverride symbols to AST for kind in [nskConst, nskType, nskProc]: - nimState.addOverrideFinal(kind) + gState.addOverrideFinal(kind) -proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode = +proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = # Create an nkConstDef PNode # # If `fname` or `fval` are set, use them as name and val @@ -134,28 +134,28 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode fname else: # node[0] = identifier = const name - nimState.getNodeVal(node.getAtom()) + gState.getNodeVal(node.getAtom()) - name = nimState.getIdentifier(origname, nskConst) - info = nimState.getLineInfo(node) - ident = nimState.getIdent(name, info) + name = gState.getIdentifier(origname, nskConst) + info = gState.getLineInfo(node) + ident = gState.getIdent(name, info) # node[1] = preproc_arg = value val = if fval.nBl: fval else: - nimState.getNodeVal(node[1]) + gState.getNodeVal(node[1]) valident = - nimState.getLit(val) + gState.getLit(val) if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, nskConst) + result = gState.getOverrideOrSkip(node, origname, nskConst) elif valident.kind in {nkCharLit .. nkStrLit} or (valident.kind == nkStmtList and valident.len > 0 and valident[0].kind in {nkCharLit .. nkStrLit}): - if nimState.addNewIdentifer(name): + if gState.addNewIdentifer(name): # const X* = Y # # nkConstDef( @@ -175,11 +175,11 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode else: result.add valident else: - necho &"# const '{origname}' is duplicate, skipped" + gecho &"# const '{origname}' is duplicate, skipped" else: - necho &"# const '{origname}' has invalid value '{val}'" + gecho &"# const '{origname}' has invalid value '{val}'" -proc addConst(nimState: NimState, node: TSNode) = +proc addConst(gState: State, node: TSNode) = # Add a const to the AST # # #define X Y @@ -189,25 +189,25 @@ proc addConst(nimState: NimState, node: TSNode) = # (preproc_arg) # ) decho("addConst()") - nimState.printDebug(node) + gState.printDebug(node) if node[0].getName() == "identifier" and node[1].getName() == "preproc_arg": let - constDef = nimState.newConstDef(node) + constDef = gState.newConstDef(node) if not constDef.isNil: # nkConstSection.add - nimState.constSection.add constDef - nimState.constIdentifiers.incl constDef.getIdentName() + gState.constSection.add constDef + gState.constIdentifiers.incl constDef.getIdentName() - nimState.printDebug(constDef) + gState.printDebug(constDef) -proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, value: PNode = nil) = +proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: PNode = nil) = # Add pragma to an existing nkPragma tree let - pinfo = nimState.getLineInfo(node.getAtom()) - pident = nimState.getIdent(name, pinfo, exported = false) + pinfo = gState.getLineInfo(node.getAtom()) + pident = gState.getIdent(name, pinfo, exported = false) if value.isNil: pragma.add pident @@ -218,17 +218,17 @@ proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, name: string, va colExpr.add value pragma.add colExpr -proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: seq[string]) = +proc addPragma(gState: State, node: TSNode, pragma: PNode, pragmas: seq[string]) = # Add sequence of pragmas to an existing nkPragma tree for name in pragmas: - nimState.addPragma(node, pragma, name) + gState.addPragma(node, pragma, name) -proc addPragma(nimState: NimState, node: TSNode, pragma: PNode, pragmas: OrderedTable[string, PNode]) = +proc addPragma(gState: State, node: TSNode, pragma: PNode, pragmas: OrderedTable[string, PNode]) = # Add a table of name:value pragmas to an existing nkPragma tree for name, value in pragmas.pairs: - nimState.addPragma(node, pragma, name, value) + gState.addPragma(node, pragma, name, value) -proc newPragma(nimState: NimState, node: TSNode, name: string, value: PNode = nil): PNode = +proc newPragma(gState: State, node: TSNode, name: string, value: PNode = nil): PNode = # Create nkPragma tree for name:value # # {.name1, name2: value2.} @@ -241,14 +241,14 @@ proc newPragma(nimState: NimState, node: TSNode, name: string, value: PNode = ni # ) # ) result = newNode(nkPragma) - nimState.addPragma(node, result, name, value) + gState.addPragma(node, result, name, value) -proc newPragma(nimState: NimState, node: TSNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = +proc newPragma(gState: State, node: TSNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = # Create nkPragma tree for multiple name:value result = newNode(nkPragma) - nimState.addPragma(node, result, pragmas) + gState.addPragma(node, result, pragmas) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, name: string, value: PNode = nil): PNode = +proc newPragmaExpr(gState: State, node: TSNode, ident: PNode, name: string, value: PNode = nil): PNode = # Create nkPragmaExpr tree for name:value # # nkPragmaExpr( @@ -266,15 +266,15 @@ proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, name: string, # ) result = newNode(nkPragmaExpr) result.add ident - result.add nimState.newPragma(node, name, value) + result.add gState.newPragma(node, name, value) -proc newPragmaExpr(nimState: NimState, node: TSNode, ident: PNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = +proc newPragmaExpr(gState: State, node: TSNode, ident: PNode, pragmas: seq[string] | OrderedTable[string, PNode]): PNode = # Create nkPragmaExpr tree for multiple name:value result = newNode(nkPragmaExpr) result.add ident - result.add nimState.newPragma(node, pragmas) + result.add gState.newPragma(node, pragmas) -proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[], istype = false): PNode = +proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: seq[string] = @[], istype = false): PNode = # Create nkTypeDef PNode with first ident if `nskType` # Create nkIdentDefs PNode with first ident if `nskVar` # Create an nkPostfix node for `nskProc` @@ -287,9 +287,9 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra (tname, torigname, info) = if not atom.isNil: - nimState.getNameInfo(node.getAtom(), kind) + gState.getNameInfo(node.getAtom(), kind) else: - ("", "", nimState.getLineInfo(node)) + ("", "", gState.getLineInfo(node)) origname = if fname.nBl: @@ -300,16 +300,16 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra # Process name if forced, getNameInfo() already runs getIdentifier() name = if fname.nBl: - nimState.getIdentifier(fname, kind) + gState.getIdentifier(fname, kind) else: tname - ident = nimState.getIdent(name, info) + ident = gState.getIdent(name, info) if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, kind) - elif nimState.addNewIdentifer(name): + result = gState.getOverrideOrSkip(node, origname, kind) + elif gState.addNewIdentifer(name): if kind == nskType: # type name* = # @@ -341,24 +341,24 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra # ) var pragmas = - if nimState.includeHeader: + if gState.isIncludeHeader(): # Need to add header and importc if istype and name == origname: # Need to add impShort since neither struct/union nor name change - pragmas & nimState.impShort + pragmas & gState.impShort else: # Add header shortcut, additional pragmas added later - pragmas & (nimState.impShort & "H") + pragmas & (gState.impShort & "H") else: pragmas prident = if pragmas.nBl: - nimState.newPragmaExpr(node, ident, pragmas) + gState.newPragmaExpr(node, ident, pragmas) else: ident - if nimState.includeHeader: + if gState.isIncludeHeader(): if not istype or name != origname: # Add importc pragma since either struct/union or name changed let @@ -370,7 +370,7 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra "struct " else: "" - nimState.addPragma(node, prident[1], "importc", newStrNode(nkStrLit, &"{uors}{origname}")) + gState.addPragma(node, prident[1], "importc", newStrNode(nkStrLit, &"{uors}{origname}")) result = newNode(nkTypeDef) result.add prident @@ -398,18 +398,18 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra prident: PNode if name != origname: # Add importc pragma since name changed - prident = nimState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) - if nimState.includeHeader(): + prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) + if gState.isIncludeHeader(): # Add header - nimState.addPragma(node, prident[1], nimState.impShort & "H") - elif nimState.gState.dynlib.nBl: + gState.addPragma(node, prident[1], gState.impShort & "H") + elif gState.dynlib.nBl: # Add dynlib - nimState.addPragma(node, prident[1], "dynlib", nimState.getIdent(nimState.gState.dynlib)) + gState.addPragma(node, prident[1], "dynlib", gState.getIdent(gState.dynlib)) else: # Only need impShort since no name change - prident = nimState.newPragmaExpr(node, ident, nimState.impShort) + prident = gState.newPragmaExpr(node, ident, gState.impShort) if pragmas.nBl: - nimState.addPragma(node, prident[1], pragmas) + gState.addPragma(node, prident[1], pragmas) prident result = newNode(nkIdentDefs) @@ -425,11 +425,11 @@ proc newXIdent(nimState: NimState, node: TSNode, kind = nskType, fname = "", pra # No pragmas here since proc pragmas are elsewhere in the AST result = ident - nimState.identifierNodes[name] = result + gState.identifierNodes[name] = result else: - necho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) + gecho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) -proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = +proc newPtrTree(gState: State, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring @@ -442,7 +442,7 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = ptname = getPtrType(tname) if tname != ptname: # If Nim type available, use that ident - result = nimState.getIdent(ptname, typ.info, exported = false) + result = gState.getIdent(ptname, typ.info, exported = false) # One ptr reduced count -= 1 if count > 0: @@ -466,18 +466,18 @@ proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = parent.add result result = nresult -proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNode = +proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input # # If `size` is nil, create UncheckedArray[typ] let - info = nimState.getLineInfo(node.getAtom()) + info = gState.getLineInfo(node.getAtom()) tname = if size.isNil: "UncheckedArray" else: "array" - ident = nimState.getIdent(tname, info, exported = false) + ident = gState.getIdent(tname, info, exported = false) # array[size, typ] # @@ -492,10 +492,10 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode = nil): PNo result.add size result.add typ -proc getTypeArray(nimState: NimState, node: TSNode, tident: PNode, name: string): PNode -proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode +proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNode +proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode -iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = +iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field # # For proc, param should not be `exported` @@ -534,7 +534,7 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So let # node[start] - param type - (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + (tname0, _, tinfo) = gState.getNameInfo(node[start].getAtom(), nskType, parent = name) # Override type name tname = @@ -543,7 +543,7 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So else: tname0 - tident = nimState.getIdent(tname, tinfo, exported = false) + tident = gState.getIdent(tname, tinfo, exported = false) # Skip qualifiers after type while start < node.len - 1 and node[start+1].getName() == "type_qualifier": @@ -559,7 +559,7 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So if tname != "object": let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported) + pident = gState.getIdent(pname, tinfo, exported) result.add pident result.add tident result.add newNode(nkEmpty) @@ -589,22 +589,22 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So # int func(char *, int **); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported) + pident = gState.getIdent(pname, tinfo, exported) acount = node[i].getXCount("abstract_pointer_declarator") result.add pident - result.add nimState.newPtrTree(acount, tident) + result.add gState.newPtrTree(acount, tident) result.add newNode(nkEmpty) else: # Named param, simple type let - (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) + (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) # Bitfield support - typedef struct { int field: 1; }; prident = if node.len > i and node[i + 1].getName() == "bitfield_clause": - nimState.newPragmaExpr(node, pident, "bitsize", - newIntNode(nkIntLit, parseInt(nimState.getNodeVal(node[i + 1].getAtom())))) + gState.newPragmaExpr(node, pident, "bitsize", + newIntNode(nkIntLit, parseInt(gState.getNodeVal(node[i + 1].getAtom())))) else: pident @@ -612,17 +612,17 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So result.add prident if count > 0: - result.add nimState.newPtrTree(count, tident) + result.add gState.newPtrTree(count, tident) else: result.add tident result.add newNode(nkEmpty) elif not fdecl.isNil: # Named param, function pointer let - (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) + (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeProc(name, node[i], node[start]) + result.add gState.getTypeProc(name, node[i], node[start]) result.add newNode(nkEmpty) elif not afdecl.isNil: # Only for proc with no named param with function pointer type @@ -631,25 +631,25 @@ iterator newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: So # int func(int (*)(int *)); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported) - procTy = nimState.getTypeProc(name, node[i], node[start]) + pident = gState.getIdent(pname, tinfo, exported) + procTy = gState.getTypeProc(name, node[i], node[start]) result.add pident result.add procTy result.add newNode(nkEmpty) elif not adecl.isNil: # Named param, array type let - (pname, _, pinfo) = nimState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported) + (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeArray(node[i], tident, name) + result.add gState.getTypeArray(node[i], tident, name) result.add newNode(nkEmpty) else: result = nil yield result -proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = +proc newFormalParams(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkFormalParams tree for specified params and return type # # proc(pname: ptyp ..): rtyp @@ -669,11 +669,11 @@ proc newFormalParams(nimState: NimState, name: string, node: TSNode, rtyp: PNode for i in 0 ..< node.len: if node[i].getName() == "parameter_declaration": # Add nkIdentDefs for each param - for param in nimState.newIdentDefs(name, node[i], i, exported = false): + for param in gState.newIdentDefs(name, node[i], i, exported = false): if not param.isNil: result.add param -proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = +proc newProcTy(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkProcTy tree for specified proc type # proc(pname: ptyp ..): rtyp @@ -688,15 +688,15 @@ proc newProcTy(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNo # nkPragma(...) # ) result = newNode(nkProcTy) - result.add nimState.newFormalParams(name, node, rtyp) - result.add nimState.newPragma(node, nimState.gState.convention) + result.add gState.newFormalParams(name, node, rtyp) + result.add gState.newPragma(node, gState.convention) # Add varargs if ... if node.getVarargs(): - nimState.addPragma(node, result[^1], "varargs") + gState.addPragma(node, result[^1], "varargs") -proc processNode(nimState: NimState, node: TSNode): bool -proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = +proc processNode(gState: State, node: TSNode): bool +proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object if not node.isNil: # fname*: ftyp @@ -722,14 +722,14 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = if not fdecl.isNil: # Nested struct / union ( - nimState.processNode(fdecl.tsNodeParent()), - nimState.typeSection[^1].getIdentName() + gState.processNode(fdecl.tsNodeParent()), + gState.typeSection[^1].getIdentName() ) elif not edecl.isNil: # Nested enum ( - nimState.processNode(edecl.tsNodeParent()), - $nimState.enumSection[^1][0][1] + gState.processNode(edecl.tsNodeParent()), + $gState.enumSection[^1][0][1] ) else: (true, "") @@ -738,11 +738,11 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = return nil # Add nkIdentDefs for each field - for field in nimState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): + for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: result.add field -proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = +proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object # # If `typeDef` is set, use it instead of creating new PNode @@ -772,7 +772,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if not node.firstChildInTree("field_declaration_list").isNil and node.tsNodeParent().getName() == "field_declaration": # If nested struct / union without a name - nimState.getUniqueIdentifier( + gState.getUniqueIdentifier( if union: "Union" else: "Type" ) else: @@ -782,7 +782,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname typeDef = if typeDef.isNil: - nimState.newXIdent(node, fname = fname, pragmas = pragmas, istype = istype) + gState.newXIdent(node, fname = fname, pragmas = pragmas, istype = istype) else: typeDef @@ -841,7 +841,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if not fdlist.isNil and fdlist.len > 0: # Add fields to object if present let - fields = nimState.newRecListTree(name, fdlist) + fields = gState.newRecListTree(name, fdlist) if fields.isNil: return obj.add fields @@ -854,16 +854,16 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if pragmas.nBl and typeDefExisting: if typeDef[0].kind != nkPragmaExpr: let - npexpr = nimState.newPragmaExpr(node, typedef[0], pragmas) + npexpr = gState.newPragmaExpr(node, typedef[0], pragmas) typedef[0] = npexpr else: # includeHeader already added impShort in newXIdent() - nimState.addPragma(node, typeDef[0][1], pragmas) + gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add - nimState.typeSection.add typeDef + gState.typeSection.add typeDef - nimState.printDebug(typeDef) + gState.printDebug(typeDef) else: # Forward declaration case let @@ -871,25 +871,25 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname if not fdlist.isNil and fdlist.len > 0: # Current node has fields let - origname = nimState.getNodeVal(node.getAtom()) + origname = gState.getNodeVal(node.getAtom()) # Fix issue #185 name = if origname.nBl: - nimState.getIdentifier(origname, nskType) + gState.getIdentifier(origname, nskType) else: "" - if name.nBl and nimState.identifierNodes.hasKey(name): + if name.nBl and gState.identifierNodes.hasKey(name): let - def = nimState.identifierNodes[name] + def = gState.identifierNodes[name] # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and def[2].kind == nkObjectTy and def[2].len == 3 and def[2][2].kind == nkEmpty: # Add fields to existing object let - fields = nimState.newRecListTree(name, fdlist) + fields = gState.newRecListTree(name, fdlist) if fields.isNil: return def[2][2] = fields @@ -899,14 +899,14 @@ proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname def[0][1].kind == nkPragma and def[0][1].len > 0: for i in 0 ..< def[0][1].len: if $def[0][1][i] == "incompleteStruct": - def[0][1][i] = nimState.getIdent( - "bycopy", nimState.getLineInfo(node.getAtom()), + def[0][1][i] = gState.getIdent( + "bycopy", gState.getLineInfo(node.getAtom()), exported = false ) - nimState.printDebug(def) + gState.printDebug(def) -proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = +proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = # Add a type of a specified type # # If `ftname` is set, use it as the type name @@ -918,14 +918,14 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # Add a type of a specific type let # node[i] = identifer = name - typeDef = nimState.newXIdent(node[i], istype = true) + typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: let name = typeDef.getIdentName() # node[start] = identifier = type name - (tname0, _, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = name) + (tname0, _, tinfo) = gState.getNameInfo(node[start].getAtom(), nskType, parent = name) # Override type name tname = @@ -934,7 +934,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = else: tname0 - ident = nimState.getIdent(tname, tinfo, exported = false) + ident = gState.getIdent(tname, tinfo, exported = false) # node[i] could have nested pointers count = node[i].getPtrCount() @@ -943,7 +943,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = if name != tname: if count > 0: # If pointers - typeDef.add nimState.newPtrTree(count, ident) + typeDef.add gState.newPtrTree(count, ident) else: typeDef.add ident @@ -961,13 +961,13 @@ proc addTypeTyped(nimState: NimState, node: TSNode, ftname = "", offset = 0) = # ) # nkTypeSection.add - nimState.typeSection.add typeDef + gState.typeSection.add typeDef - nimState.printDebug(typeDef) + gState.printDebug(typeDef) else: - nimState.addTypeObject(node, typeDef = typeDef, istype = true) + gState.addTypeObject(node, typeDef = typeDef, istype = true) -proc getTypeArray(nimState: NimState, node: TSNode, tident: PNode, name: string): PNode = +proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNode = # Create array type tree # # `tident` is type PNode @@ -1003,45 +1003,45 @@ proc getTypeArray(nimState: NimState, node: TSNode, tident: PNode, name: string) if tcount > 0: # If pointers - result = nimState.newPtrTree(tcount, result) + result = gState.newPtrTree(tcount, result) for i in 0 ..< acount: if cnode.len == 2: # type name[X] => array[X, type] let # Size of array could be a Nim expression - size = nimState.getLit(nimState.getNodeVal(cnode[1]), expression = true) + size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) if size.kind != nkNilLit: - result = nimState.newArrayTree(cnode, result, size) + result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] elif cnode.len == 1: # type name[] = UncheckedArray[type] - result = nimState.newArrayTree(cnode, result) + result = gState.newArrayTree(cnode, result) cnode = cnode[0] if ncount > 0: - result = nimState.newPtrTree(ncount, result) + result = gState.newPtrTree(ncount, result) -proc addTypeArray(nimState: NimState, node: TSNode) = +proc addTypeArray(gState: State, node: TSNode) = # Add a type of array type decho("addTypeArray()") let start = getStartAtom(node) # node[start] = identifier = type name - (tname, _, info) = nimState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") - tident = nimState.getIdent(tname, info, exported = false) + (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") + tident = gState.getIdent(tname, info, exported = false) # Could have multiple types, comma separated for i in start+1 ..< node.len: let # node[i] = identifer = name - typeDef = nimState.newXIdent(node[i], istype = true) + typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: let name = typeDef.getIdentName() - typ = nimState.getTypeArray(node[i], tident, name) + typ = gState.getTypeArray(node[i], tident, name) typeDef.add typ @@ -1065,17 +1065,17 @@ proc addTypeArray(nimState: NimState, node: TSNode) = # ) # nkTypeSection.add - nimState.typeSection.add typeDef + gState.typeSection.add typeDef - nimState.printDebug(typeDef) + gState.printDebug(typeDef) -proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = +proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode = # Create proc type tree # # `rnode` is the return type let # rnode = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(rnode.getAtom(), nskType, parent = name) + (rname, _, rinfo) = gState.getNameInfo(rnode.getAtom(), nskType, parent = name) # Parameter list plist = node.anyChildInTree("parameter_list") @@ -1113,16 +1113,16 @@ proc getTypeProc(nimState: NimState, name: string, node, rnode: TSNode): PNode = # void (*func)(..) newNode(nkEmpty) else: - nimState.getIdent(rname, rinfo, exported = false) + gState.getIdent(rname, rinfo, exported = false) if tcount > 0: - retType = nimState.newPtrTree(tcount, retType) + retType = gState.newPtrTree(tcount, retType) # Proc with return type and params - result = nimState.newProcTy(name, plist, retType) + result = gState.newProcTy(name, plist, retType) if ncount > 1: - result = nimState.newPtrTree(ncount-1, result) + result = gState.newPtrTree(ncount-1, result) -proc addTypeProc(nimState: NimState, node: TSNode) = +proc addTypeProc(gState: State, node: TSNode) = # Add a type of proc type decho("addTypeProc()") let @@ -1135,13 +1135,13 @@ proc addTypeProc(nimState: NimState, node: TSNode) = for i in start+1 ..< node.len: let # node[i] = identifier = name - typeDef = nimState.newXIdent(node[i], istype = true) + typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: let name = typeDef.getIdentName() - procTy = nimState.getTypeProc(name, node[i], rnode) + procTy = gState.getTypeProc(name, node[i], rnode) typeDef.add procTy @@ -1174,13 +1174,13 @@ proc addTypeProc(nimState: NimState, node: TSNode) = # ) # nkTypeSection.add - nimState.typeSection.add typeDef + gState.typeSection.add typeDef - nimState.printDebug(typeDef) + gState.printDebug(typeDef) -proc addType(nimState: NimState, node: TSNode, union = false) = +proc addType(gState: State, node: TSNode, union = false) = decho("addType()") - nimState.printDebug(node) + gState.printDebug(node) if node.getName() in ["struct_specifier", "union_specifier"]: # struct X; @@ -1212,13 +1212,13 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # (field_declaration ...) # ) decho("addType(): case 1") - nimState.addTypeObject(node, union = union) + gState.addTypeObject(node, union = union) elif node.getName() == "type_definition": if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") if (fdlist.isNil or (not fdlist.isNil and fdlist.Bl)) and - nimState.getNodeVal(node[1]) == "": + gState.getNodeVal(node[1]) == "": # typedef struct X; # # (type_definition @@ -1238,7 +1238,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # (type_definition = "") # ) decho("addType(): case 2") - nimState.addTypeObject(node[0], union = union) + gState.addTypeObject(node[0], union = union) else: let fdecl = node[1].anyChildInTree("function_declarator") @@ -1262,7 +1262,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # ) # ) decho("addType(): case 3") - nimState.addTypeTyped(node) + gState.addTypeTyped(node) elif not fdecl.isNil: # typedef X (*Y)(a1, a2, a3); # typedef X *(*Y)(a1, a2, a3); @@ -1293,7 +1293,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # ) # ) decho("addType(): case 4") - nimState.addTypeProc(node) + gState.addTypeProc(node) elif not adecl.isNil: # typedef struct X Y[a][..]; # typedef struct X *Y[a][..]; @@ -1316,7 +1316,7 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # ) # ) decho("addType(): case 5") - nimState.addTypeArray(node) + gState.addTypeArray(node) else: if node.firstChildInTree("field_declaration_list").isNil: # typedef struct X { .. } Y, *Z; @@ -1344,11 +1344,11 @@ proc addType(nimState: NimState, node: TSNode, union = false) = # First add struct as object decho("addType(): case 6") - nimState.addTypeObject(node[0], union = union) + gState.addTypeObject(node[0], union = union) - if node.len > 1 and nimState.getNodeVal(node[1]) != "": + if node.len > 1 and gState.getNodeVal(node[1]) != "": # Add any additional names - nimState.addTypeTyped(node) + gState.addTypeTyped(node) else: # Same as above except unnamed struct # @@ -1362,20 +1362,20 @@ proc addType(nimState: NimState, node: TSNode, union = false) = name = "" for i in 1 ..< node.len: if node[i].getName() == "type_identifier": - name = nimState.getNodeVal(node[i].getAtom()) + name = gState.getNodeVal(node[i].getAtom()) name # Now add struct as object with specified name - nimState.addTypeObject(node[0], fname = name, istype = true, union = union) + gState.addTypeObject(node[0], fname = name, istype = true, union = union) if name.nBl: # Add any additional names - nimState.addTypeTyped(node, ftname = name, offset = 1) + gState.addTypeTyped(node, ftname = name, offset = 1) -proc addEnum(nimState: NimState, node: TSNode) = +proc addEnum(gState: State, node: TSNode) = decho("addEnum()") - nimState.printDebug(node) + gState.printDebug(node) let enumlist = node.anyChildInTree("enumerator_list") @@ -1388,29 +1388,29 @@ proc addEnum(nimState: NimState, node: TSNode) = if node.getAtom().getName() == "type_identifier": # [typedef] enum X {} Y; # Use X as name - origname = nimState.getNodeVal(node.getAtom()) + origname = gState.getNodeVal(node.getAtom()) elif node.getName() == "type_definition" and node.len > 1: # typedef enum {} Y; # Use Y as name - origname = nimState.getNodeVal(node[1].getAtom()) + origname = gState.getNodeVal(node[1].getAtom()) offset = 1 if origname.nBl: - name = nimState.getIdentifier(origname, nskType) + name = gState.getIdentifier(origname, nskType) else: # enum {}; # Nameless so create a name - name = nimState.getUniqueIdentifier("Enum") + name = gState.getUniqueIdentifier("Enum") if name.Bl: # Name skipped or overridden since blank let - eoverride = nimState.getOverrideOrSkip(node, origname, nskType) + eoverride = gState.getOverrideOrSkip(node, origname, nskType) if not eoverride.isNil: - nimState.typeSection.add eoverride - elif nimState.addNewIdentifer(name): + gState.typeSection.add eoverride + elif gState.addNewIdentifer(name): # Add enum definition and helpers - nimState.enumSection.add nimState.parseString(&"defineEnum({name})") + gState.enumSection.add gState.parseString(&"defineEnum({name})") # Create const for fields var @@ -1421,7 +1421,7 @@ proc addEnum(nimState: NimState, node: TSNode) = if en.getName() == "comment": continue let - fname = nimState.getIdentifier(nimState.getNodeVal(en.getAtom()), nskEnumField) + fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField) if fname.nBl: var fval = "" @@ -1434,10 +1434,10 @@ proc addEnum(nimState: NimState, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & nimState.getNimExpression(nimState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] fnames.incl fname @@ -1445,25 +1445,25 @@ proc addEnum(nimState: NimState, node: TSNode) = # Add fields to list of consts after processing enum so that we don't cast # enum field to itself - nimState.constIdentifiers.incl fnames + gState.constIdentifiers.incl fnames # Add other names if node.getName() == "type_definition" and node.len > 1: - nimState.addTypeTyped(node, ftname = name, offset = offset) + gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(nimState: NimState, node, rnode: TSNode) = +proc addProcVar(gState: State, node, rnode: TSNode) = # Add a proc variable decho("addProcVar()") let # node = identifier = name - identDefs = nimState.newXIdent(node, kind = nskVar, istype = true) + identDefs = gState.newXIdent(node, kind = nskVar, istype = true) if not identDefs.isNil: let name = identDefs.getIdentName() - # origname = nimState.getNodeVal(node.getAtom()) + # origname = gState.getNodeVal(node.getAtom()) - procTy = nimState.getTypeProc(name, node, rnode) + procTy = gState.getTypeProc(name, node, rnode) identDefs.add procTy identDefs.add newNode(nkEmpty) @@ -1505,11 +1505,11 @@ proc addProcVar(nimState: NimState, node, rnode: TSNode) = # ) # nkVarSection.add - nimState.varSection.add identDefs + gState.varSection.add identDefs - nimState.printDebug(identDefs) + gState.printDebug(identDefs) -proc addProc(nimState: NimState, node, rnode: TSNode) = +proc addProc(gState: State, node, rnode: TSNode) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1517,19 +1517,19 @@ proc addProc(nimState: NimState, node, rnode: TSNode) = decho("addProc()") let # node = identifier = name - ident = nimState.newXIdent(node, kind = nskProc) + ident = gState.newXIdent(node, kind = nskProc) if not ident.isNil: let # Only need the ident tree, not nkTypeDef parent name = ident.getIdentName() - origname = nimState.getNodeVal(node.getAtom()) + origname = gState.getNodeVal(node.getAtom()) # node could have nested pointers tcount = node.getPtrCount() # rnode = identifier = return type name - (rname, _, rinfo) = nimState.getNameInfo(rnode.getAtom(), nskType, parent = name) + (rname, _, rinfo) = gState.getNameInfo(rnode.getAtom(), nskType, parent = name) # Parameter list plist = node.anyChildInTree("parameter_list") @@ -1574,56 +1574,56 @@ proc addProc(nimState: NimState, node, rnode: TSNode) = # void func(..) newNode(nkEmpty) else: - nimState.getIdent(rname, rinfo, exported = false) + gState.getIdent(rname, rinfo, exported = false) if tcount > 0: - retType = nimState.newPtrTree(tcount, retType) + retType = gState.newPtrTree(tcount, retType) # Proc with return type and params - procDef.add nimState.newFormalParams(name, plist, retType) + procDef.add gState.newFormalParams(name, plist, retType) # Pragmas let prident = if name != origname: # Explicit {.importc: "origname".} - nimState.newPragma(node, "importc", newStrNode(nkStrLit, origname)) + gState.newPragma(node, "importc", newStrNode(nkStrLit, origname)) else: # {.impnameC.} shortcut - nimState.newPragma(node, nimState.impShort & "C") + gState.newPragma(node, gState.impShort & "C") # Detect ... and add {.varargs.} pvarargs = plist.getVarargs() # Need {.convention.} and {.header.} if applicable if name != origname: - if nimState.includeHeader(): + if gState.isIncludeHeader(): # {.impnameHC.} shortcut - nimState.addPragma(node, prident, nimState.impShort & "HC") + gState.addPragma(node, prident, gState.impShort & "HC") else: # {.convention.} - nimState.addPragma(node, prident, nimState.gState.convention) + gState.addPragma(node, prident, gState.convention) - if nimState.gState.dynlib.nBl: + if gState.dynlib.nBl: # {.dynlib.} for DLLs - nimState.addPragma(node, prident, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + gState.addPragma(node, prident, "dynlib", gState.getIdent(gState.dynlib)) if pvarargs: # Add {.varargs.} for ... - nimState.addPragma(node, prident, "varargs") + gState.addPragma(node, prident, "varargs") procDef.add prident procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) # nkProcSection.add - nimState.procSection.add procDef + gState.procSection.add procDef - nimState.printDebug(procDef) + gState.printDebug(procDef) -proc addDecl(nimState: NimState, node: TSNode) = +proc addDecl(gState: State, node: TSNode) = # Add a declaration decho("addDecl()") - nimState.printDebug(node) + gState.printDebug(node) let start = getStartAtom(node) @@ -1633,58 +1633,58 @@ proc addDecl(nimState: NimState, node: TSNode) = # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - nimState.addProcVar(node[i], node[start]) + gState.addProcVar(node[i], node[start]) else: # proc - nimState.addProc(node[i], node[start]) + gState.addProc(node[i], node[start]) else: # Regular var discard -proc addDef(nimState: NimState, node: TSNode) = +proc addDef(gState: State, node: TSNode) = # Wrap static inline definition if {.header.} mode is specified # # Without {.header.} the definition will not be available to the C compiler # and will fail at link time decho("addDef()") - nimState.printDebug(node) + gState.printDebug(node) let start = getStartAtom(node) if node[start+1].getName() == "function_declarator": - if nimState.includeHeader(): - nimState.addProc(node[start+1], node[start]) + if gState.isIncludeHeader(): + gState.addProc(node[start+1], node[start]) else: - necho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % - nimState.getNodeVal(node[start+1].getAtom()) + gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % + gState.getNodeVal(node[start+1].getAtom()) -proc processNode(nimState: NimState, node: TSNode): bool = +proc processNode(gState: State, node: TSNode): bool = result = true case node.getName() of "preproc_def": - nimState.addConst(node) + gState.addConst(node) of "type_definition": if node.len > 0 and node[0].getName() == "enum_specifier": - nimState.addEnum(node) + gState.addEnum(node) elif node.len > 0 and node[0].getName() == "union_specifier": - nimState.addType(node, union = true) + gState.addType(node, union = true) else: - nimState.addType(node) + gState.addType(node) of "struct_specifier": - nimState.addType(node) + gState.addType(node) of "union_specifier": - nimState.addType(node, union = true) + gState.addType(node, union = true) of "enum_specifier": - nimState.addEnum(node) + gState.addEnum(node) of "declaration": - nimState.addDecl(node) + gState.addDecl(node) of "function_definition": - nimState.addDef(node) + gState.addDef(node) else: # Unknown result = false -proc searchTree(nimState: NimState, root: TSNode) = +proc searchTree(gState: State, root: TSNode) = # Search AST generated by tree-sitter for recognized elements var node = root @@ -1694,7 +1694,7 @@ proc searchTree(nimState: NimState, root: TSNode) = while true: if not node.isNil and depth > -1: - processed = nimState.processNode(node) + processed = gState.processNode(node) else: break @@ -1721,7 +1721,7 @@ proc searchTree(nimState: NimState, root: TSNode) = if node == root: break -proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = +proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # Create shortcut pragmas to reduce clutter var hdrPragma: PNode @@ -1730,41 +1730,41 @@ proc setupPragmas(nimState: NimState, root: TSNode, fullpath: string) = impConvPragma = newNode(nkPragma) # {.pragma: impname, importc.} - nimState.addPragma(root, impPragma, "pragma", nimState.getIdent(nimState.impShort)) - nimState.addPragma(root, impPragma, "importc") + gState.addPragma(root, impPragma, "pragma", gState.getIdent(gState.impShort)) + gState.addPragma(root, impPragma, "importc") - if nimState.includeHeader(): + if gState.isIncludeHeader(): # Path to header const - nimState.constSection.add nimState.newConstDef( - root, fname = nimState.currentHeader, fval = '"' & fullpath & '"') + gState.constSection.add gState.newConstDef( + root, fname = gState.currentHeader, fval = '"' & fullpath & '"') # {.pragma: impnameH, header: "xxx".} for types when name != origname - hdrPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "H")) - nimState.addPragma(root, hdrPragma, "header", nimState.getIdent(nimState.currentHeader)) + hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "H")) + gState.addPragma(root, hdrPragma, "header", gState.getIdent(gState.currentHeader)) # Add {.impnameH.} to {.impname.} - nimState.addPragma(root, impPragma, nimState.impShort & "H") + gState.addPragma(root, impPragma, gState.impShort & "H") # {.pragma: impnameHC, impnameH, convention.} for procs when name != origname - hdrConvPragma = nimState.newPragma(root, "pragma", nimState.getIdent(nimState.impShort & "HC")) - nimState.addPragma(root, hdrConvPragma, nimState.impShort & "H") - nimState.addPragma(root, hdrConvPragma, nimState.gState.convention) + hdrConvPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "HC")) + gState.addPragma(root, hdrConvPragma, gState.impShort & "H") + gState.addPragma(root, hdrConvPragma, gState.convention) # {.pragma: impnameC, impname, convention.} for procs - nimState.addPragma(root, impConvPragma, "pragma", nimState.getIdent(nimState.impShort & "C")) - nimState.addPragma(root, impConvPragma, nimState.impShort) - nimState.addPragma(root, impConvPragma, nimState.gState.convention) + gState.addPragma(root, impConvPragma, "pragma", gState.getIdent(gState.impShort & "C")) + gState.addPragma(root, impConvPragma, gState.impShort) + gState.addPragma(root, impConvPragma, gState.convention) - if nimState.gState.dynlib.nBl: + if gState.dynlib.nBl: # {.dynlib.} for DLLs - nimState.addPragma(root, impConvPragma, "dynlib", nimState.getIdent(nimState.gState.dynlib)) + gState.addPragma(root, impConvPragma, "dynlib", gState.getIdent(gState.dynlib)) # Add all pragma shortcuts to output if not hdrPragma.isNil: - nimState.pragmaSection.add hdrPragma - nimState.pragmaSection.add hdrConvPragma - nimState.pragmaSection.add impPragma - nimState.pragmaSection.add impConvPragma + gState.pragmaSection.add hdrPragma + gState.pragmaSection.add hdrConvPragma + gState.pragmaSection.add impPragma + gState.pragmaSection.add impConvPragma proc printNimHeader*(gState: State) = # Top level output with context info @@ -1780,49 +1780,47 @@ import nimterop/types proc printNim*(gState: State, fullpath: string, root: TSNode) = # Generate Nim from tree-sitter AST root node let - nimState = new(NimState) fp = fullpath.replace("\\", "/") # Track identifiers already rendered and corresponding PNodes - nimState.identifiers = newTable[string, string]() - nimState.identifierNodes = newTable[string, PNode]() + gState.identifiers = newTable[string, string]() + gState.identifierNodes = newTable[string, PNode]() # toast objects - nimState.gState = gState - nimState.currentHeader = getCurrentHeader(fullpath) - nimState.impShort = nimState.currentHeader.replace("header", "imp") - nimState.sourceFile = fullpath + gState.currentHeader = getCurrentHeader(fullpath) + gState.impShort = gState.currentHeader.replace("header", "imp") + gState.sourceFile = fullpath # Nim compiler objects - nimState.identCache = newIdentCache() - nimState.config = newConfigRef() - nimstate.graph = newModuleGraph(nimState.identCache, nimState.config) + gState.identCache = newIdentCache() + gState.config = newConfigRef() + gState.graph = newModuleGraph(gState.identCache, gState.config) # Initialize all section PNodes - nimState.constSection = newNode(nkConstSection) - nimState.enumSection = newNode(nkStmtList) - nimState.pragmaSection = newNode(nkStmtList) - nimState.procSection = newNode(nkStmtList) - nimState.typeSection = newNode(nkTypeSection) - nimState.varSection = newNode(nkVarSection) + gState.constSection = newNode(nkConstSection) + gState.enumSection = newNode(nkStmtList) + gState.pragmaSection = newNode(nkStmtList) + gState.procSection = newNode(nkStmtList) + gState.typeSection = newNode(nkTypeSection) + gState.varSection = newNode(nkVarSection) # Setup pragmas - nimState.setupPragmas(root, fp) + gState.setupPragmas(root, fp) # Search root node and render Nim - nimState.searchTree(root) + gState.searchTree(root) # Add any unused cOverride symbols to output - nimState.addAllOverrideFinal() + gState.addAllOverrideFinal() # Create output to Nim using Nim compiler renderer var tree = newNode(nkStmtList) - tree.add nimState.enumSection - tree.add nimState.constSection - tree.add nimState.pragmaSection - tree.add nimState.typeSection - tree.add nimState.varSection - tree.add nimState.procSection + tree.add gState.enumSection + tree.add gState.constSection + tree.add gState.pragmaSection + tree.add gState.typeSection + tree.add gState.varSection + tree.add gState.procSection gecho tree.renderTree() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 654c7f3..1cdd2ff 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -113,32 +113,32 @@ proc checkIdentifier(name, kind, parent, origName: string) = if parent.nBl: doAssert name.nBl, &"Blank identifier, originally '{parentStr}{origName}' ({kind}), cannot be empty" -proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent=""): string = +proc getIdentifier*(gState: State, name: string, kind: NimSymKind, parent=""): string = doAssert name.nBl, "Blank identifier error" - if name notin nimState.gState.symOverride or parent.nBl: - if nimState.gState.onSymbol != nil: + if name notin gState.symOverride or parent.nBl: + if gState.onSymbol != nil: # Use onSymbol from plugin provided by user var sym = Symbol(name: name, parent: parent, kind: kind) - nimState.gState.onSymbol(sym) + gState.onSymbol(sym) result = sym.name else: result = name # Strip out --prefix from CLI if specified - for str in nimState.gState.prefix: + for str in gState.prefix: if result.startsWith(str): result = result[str.len .. ^1] # Strip out --suffix from CLI if specified - for str in nimState.gState.suffix: + for str in gState.suffix: if result.endsWith(str): result = result[0 .. ^(str.len+1)] # --replace from CLI if specified - for name, value in nimState.gState.replace.pairs: + for name, value in gState.replace.pairs: if name.len > 1 and name[0] == '@': result = result.replace(re(name[1 .. ^1]), value) else: @@ -153,58 +153,58 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent=" # Skip identifier since in symOverride result = "" -proc getUniqueIdentifier*(nimState: NimState, prefix = ""): string = +proc getUniqueIdentifier*(gState: State, prefix = ""): string = var - name = prefix & "_" & nimState.sourceFile.extractFilename().multiReplace([(".", ""), ("-", "")]) + name = prefix & "_" & gState.sourceFile.extractFilename().multiReplace([(".", ""), ("-", "")]) nimName = name[0] & name[1 .. ^1].replace("_", "").toLowerAscii count = 1 - while (nimName & $count) in nimState.identifiers: + while (nimName & $count) in gState.identifiers: count += 1 return name & $count -proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool = - if override or name notin nimState.gState.symOverride: +proc addNewIdentifer*(gState: State, name: string, override = false): bool = + if override or name notin gState.symOverride: let nimName = name[0] & name[1 .. ^1].replace("_", "").toLowerAscii - if nimState.identifiers.hasKey(nimName): - doAssert name == nimState.identifiers[nimName], + if gState.identifiers.hasKey(nimName): + doAssert name == gState.identifiers[nimName], &"Identifier '{name}' is a stylistic duplicate of identifier " & - &"'{nimState.identifiers[nimName]}', use 'cPlugin:onSymbol()' to rename" + &"'{gState.identifiers[nimName]}', use 'cPlugin:onSymbol()' to rename" result = false else: - nimState.identifiers[nimName] = name + gState.identifiers[nimName] = name result = true # Overrides related -proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = +proc getOverride*(gState: State, name: string, kind: NimSymKind): string = # Get cOverride for identifier `name` of `kind` if defined doAssert name.nBl, "Blank identifier error" - if nimState.gState.onSymbolOverride != nil: + if gState.onSymbolOverride != nil: var - nname = nimState.getIdentifier(name, kind, "Override") + nname = gState.getIdentifier(name, kind, "Override") sym = Symbol(name: nname, kind: kind) if nname.nBl: - nimState.gState.onSymbolOverride(sym) + gState.onSymbolOverride(sym) - if sym.override.nBl and nimState.addNewIdentifer(nname, override = true): + if sym.override.nBl and gState.addNewIdentifer(nname, override = true): result = sym.override if kind != nskProc: result = result.replace(re"(?m)^(.*?)$", " $1") -proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = +proc getOverrideFinal*(gState: State, kind: NimSymKind): string = # Get all unused cOverride symbols of `kind` let typ = $kind - if nimState.gState.onSymbolOverrideFinal != nil: - for i in nimState.gState.onSymbolOverrideFinal(typ): - result &= "\n" & nimState.getOverride(i, kind) + if gState.onSymbolOverrideFinal != nil: + for i in gState.onSymbolOverrideFinal(typ): + result &= "\n" & gState.getOverride(i, kind) proc getKeyword*(kind: NimSymKind): string = # Convert `kind` into a Nim keyword @@ -232,9 +232,6 @@ proc getNodeVal*(gState: State, node: TSNode): string = if not node.isNil: return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() -proc getNodeVal*(nimState: NimState, node: TSNode): string = - nimState.gState.getNodeVal(node) - proc getAtom*(node: TSNode): TSNode = if not node.isNil: # Get child node which is topmost atom @@ -428,8 +425,8 @@ proc printLisp*(gState: State, root: TSNode): string = proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") -proc printTree*(nimState: NimState, pnode: PNode, offset = ""): string = - if nimState.gState.debug and pnode.kind != nkNone: +proc printTree*(gState: State, pnode: PNode, offset = ""): string = + if gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: @@ -447,7 +444,7 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = ""): string = else: if pnode.sons.len != 0: for i in 0 ..< pnode.sons.len: - result &= nimState.printTree(pnode.sons[i], offset & " ") + result &= gState.printTree(pnode.sons[i], offset & " ") if i != pnode.sons.len - 1: result &= "," result &= "\n# " & offset & ")" @@ -456,34 +453,34 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(nimState: NimState, node: TSNode) = - if nimState.gState.debug: - necho ("Input => " & nimState.getNodeVal(node)).getCommented() & "\n" & - nimState.gState.printLisp(node).getCommented() +proc printDebug*(gState: State, node: TSNode) = + if gState.debug: + gecho ("Input => " & gState.getNodeVal(node)).getCommented() & "\n" & + gState.printLisp(node).getCommented() -proc printDebug*(nimState: NimState, pnode: PNode) = - if nimState.gState.debug: - necho ("Output => " & $pnode).getCommented() & "\n" & - nimState.printTree(pnode) +proc printDebug*(gState: State, pnode: PNode) = + if gState.debug: + gecho ("Output => " & $pnode).getCommented() & "\n" & + gState.printTree(pnode) # Compiler shortcuts -proc getDefaultLineInfo*(nimState: NimState): TLineInfo = - result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, 0, 0) +proc getDefaultLineInfo*(gState: State): TLineInfo = + result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, 0, 0) -proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = +proc getLineInfo*(gState: State, node: TSNode): TLineInfo = # Get Nim equivalent line:col info from node let - (line, col) = nimState.gState.getLineCol(node) + (line, col) = gState.getLineCol(node) - result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col) + result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, line, col) -proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode = +proc getIdent*(gState: State, name: string, info: TLineInfo, exported = true): PNode = if name.nBl: # Get ident PNode for name + info let - exp = getIdent(nimState.identCache, "*") - ident = getIdent(nimState.identCache, name) + exp = getIdent(gState.identCache, "*") + ident = getIdent(gState.identCache, name) if exported: result = newNode(nkPostfix) @@ -492,8 +489,8 @@ proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = tru else: result = newIdentNode(ident, info) -proc getIdent*(nimState: NimState, name: string): PNode = - nimState.getIdent(name, nimState.getDefaultLineInfo(), exported = false) +proc getIdent*(gState: State, name: string): PNode = + gState.getIdent(name, gState.getDefaultLineInfo(), exported = false) proc getIdentName*(node: PNode): string = if not node.isNil: @@ -503,15 +500,15 @@ proc getIdentName*(node: PNode): string = if result.Bl and node.len > 0: result = node[0].getIdentName() -proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): +proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): tuple[name, origname: string, info: TLineInfo] = # Shortcut to get identifier name and info (node value and line:col) - result.origname = nimState.getNodeVal(node) - result.name = nimState.getIdentifier(result.origname, kind, parent) + result.origname = gState.getNodeVal(node) + result.name = gState.getIdentifier(result.origname, kind, parent) if result.name.nBl: if kind == nskType: result.name = result.name.getType() - result.info = nimState.getLineInfo(node) + result.info = gState.getLineInfo(node) proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) @@ -644,7 +641,7 @@ proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = if ast.children.len == 1 and ast.children[0].name == ".": return ast.children[0] -proc getNimExpression*(nimState: NimState, expr: string, name = ""): string = +proc getNimExpression*(gState: State, expr: string, name = ""): string = # Convert C/C++ expression into Nim - cast identifiers to `name` if specified var clean = expr.multiReplace([("\n", " "), ("\r", "")]) @@ -691,8 +688,8 @@ proc getNimExpression*(nimState: NimState, expr: string, name = ""): string = if ident.nBl: # Issue #178 if ident != "_": - ident = nimState.getIdentifier(ident, nskConst, name) - if name.nBl and ident in nimState.constIdentifiers: + ident = gState.getIdentifier(ident, nskConst, name) + if name.nBl and ident in gState.constIdentifiers: ident = ident & "." & name result &= ident ident = "" @@ -708,15 +705,15 @@ proc getSplitComma*(joined: seq[string]): seq[string] = for i in joined: result = result.concat(i.split(",")) -template includeHeader*(nimState: NimState): bool = - nimState.gState.dynlib.Bl and nimState.gState.includeHeader +template isIncludeHeader*(gState: State): bool = + gState.dynlib.Bl and gState.includeHeader -proc getComments*(nimState: NimState, strip = false): string = - if not nimState.gState.nocomments and nimState.commentStr.nBl: - result = "\n" & nimState.commentStr +proc getComments*(gState: State, strip = false): string = + if not gState.nocomments and gState.commentStr.nBl: + result = "\n" & gState.commentStr if strip: result = result.replace("\n ", "\n") - nimState.commentStr = "" + gState.commentStr = "" proc dll*(path: string): string = let diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 47157d8..f159124 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -49,7 +49,7 @@ type recursive*: bool children*: seq[ref Ast] when not declared(CIMPORT): - tonim*: proc (ast: ref Ast, node: TSNode, nimState: NimState) + tonim*: proc (ast: ref Ast, node: TSNode, gState: State) regex*: Regex AstTable {.used.} = TableRef[string, seq[ref Ast]] @@ -70,7 +70,6 @@ type outputHandle*: File - NimState {.used.} = ref object # All symbols that have been declared so far indexed by nimName identifiers*: TableRef[string, string] @@ -92,8 +91,6 @@ type # Craeted symbols to generated AST - forward declaration tracking identifierNodes*: TableRef[string, PNode] - gState*: State - currentHeader*, impShort*, sourceFile*: string data*: seq[tuple[name, val: string]] @@ -113,8 +110,7 @@ template Bl(s: typed): untyped {.used.} = (s.len == 0) when not declared(CIMPORT): - export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, - nBl, Bl + export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl # Redirect output to file when required template gecho*(args: string) {.dirty.} = @@ -123,11 +119,6 @@ when not declared(CIMPORT): else: gState.outputHandle.writeLine(args) - template necho*(args: string) {.dirty.} = - when not declared(gState): - let gState = nimState.gState - gecho args - template decho*(str: untyped): untyped = - if nimState.gState.debug: - necho str.getCommented() + if gState.debug: + gecho str.getCommented() diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index 8e1d290..0d804d0 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -5,7 +5,7 @@ import regex import "."/[ast, getters, globals, lisp, treesitter/api] type - Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, nimState: NimState) {.nimcall.}]] + Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] proc getPtrType(str: string): string = result = case str: @@ -39,26 +39,26 @@ proc initGrammar(): Grammar = (preproc_arg) ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# define X Y" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# define X Y" let - name = nimState.data[0].val - nname = nimState.getIdentifier(name, nskConst) - val = nimState.data[1].val.getLit() + name = gState.data[0].val + nname = gState.getIdentifier(name, nskConst) + val = gState.data[1].val.getLit() if not nname.nBl: let - override = nimState.getOverride(name, nskConst) + override = gState.getOverride(name, nskConst) if override.nBl: - nimState.constStr &= &"{nimState.getComments()}\n{override}" + gState.constStr &= &"{gState.getComments()}\n{override}" else: - nimState.constStr &= &"{nimState.getComments()}\n # Const '{name}' skipped" - if nimState.gState.debug: - nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" - elif val.nBl and nimState.addNewIdentifer(nname): - nimState.constStr &= &"{nimState.getComments()}\n {nname}* = {val}" + gState.constStr &= &"{gState.getComments()}\n # Const '{name}' skipped" + if gState.debug: + gState.skipStr &= &"\n{gState.getNodeVal(node)}" + elif val.nBl and gState.addNewIdentifer(nname): + gState.constStr &= &"{gState.getComments()}\n {nname}* = {val}" )) let @@ -125,25 +125,25 @@ proc initGrammar(): Grammar = """ template funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen: untyped): untyped = - ptyp = nimState.getIdentifier(nimState.data[i].val, nskType, fname).getType() + ptyp = gState.getIdentifier(gState.data[i].val, nskType, fname).getType() pptr = "" - while i+1 < nimState.data.len and nimState.data[i+1].name == "pointer_declarator": + while i+1 < gState.data.len and gState.data[i+1].name == "pointer_declarator": pptr &= "ptr " i += 1 - if i+1 < nimState.data.len and nimState.data[i+1].name == "identifier": - pname = nimState.getIdentifier(nimState.data[i+1].val, nskParam, fname) + if i+1 < gState.data.len and gState.data[i+1].name == "identifier": + pname = gState.getIdentifier(gState.data[i+1].val, nskParam, fname) i += 2 else: pname = "a" & $count count += 1 i += 1 - if i < nimState.data.len and nimState.data[i].name in ["identifier", "number_literal"]: - flen = nimState.data[i].val - if nimState.data[i].name == "identifier": - flen = nimState.getIdentifier(flen, nskConst, fname) + if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: + flen = gState.data[i].val + if gState.data[i].name == "identifier": + flen = gState.getIdentifier(flen, nskConst, fname) pout &= &"{pname}: array[{flen}, {getPtrType(pptr&ptyp)}], " i += 1 @@ -172,13 +172,13 @@ proc initGrammar(): Grammar = {funcGrammar} ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# typedef X Y" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# typedef X Y" var i = 0 - typ = nimState.getIdentifier(nimState.data[i].val, nskType, "Parent").getType() + typ = gState.getIdentifier(gState.data[i].val, nskType, "Parent").getType() name = "" nname = "" tptr = "" @@ -186,8 +186,8 @@ proc initGrammar(): Grammar = pragmas: seq[string] = @[] i += 1 - while i < nimState.data.len and "pointer" in nimState.data[i].name: - case nimState.data[i].name: + while i < gState.data.len and "pointer" in gState.data[i].name: + case gState.data[i].name: of "pointer_declarator": tptr &= "ptr " i += 1 @@ -195,32 +195,32 @@ proc initGrammar(): Grammar = aptr &= "ptr " i += 1 - if i < nimState.data.len: - name = nimState.data[i].val - nname = nimState.getIdentifier(name, nskType) + if i < gState.data.len: + name = gState.data[i].val + nname = gState.getIdentifier(name, nskType) i += 1 - if nimState.includeHeader(): - pragmas.add nimState.getImportC(name, nname) + if gState.isIncludeHeader(): + pragmas.add gState.getImportC(name, nname) let - pragma = nimState.getPragma(pragmas) + pragma = gState.getPragma(pragmas) if not nname.nBl: let - override = nimState.getOverride(name, nskType) + override = gState.getOverride(name, nskType) if override.nBl: - nimState.typeStr &= &"{nimState.getComments()}\n{override}" - elif nname notin gTypeMap and typ.nBl and nimState.addNewIdentifer(nname): - if i < nimState.data.len and nimState.data[^1].name == "function_declarator": + gState.typeStr &= &"{gState.getComments()}\n{override}" + elif nname notin gTypeMap and typ.nBl and gState.addNewIdentifer(nname): + if i < gState.data.len and gState.data[^1].name == "function_declarator": var fname = nname pout, pname, ptyp, pptr = "" count = 1 flen = "" - while i < nimState.data.len: - if nimState.data[i].name == "function_declarator": + while i < gState.data.len: + if gState.data[i].name == "function_declarator": break funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) @@ -229,34 +229,34 @@ proc initGrammar(): Grammar = pout = pout[0 .. ^3] if tptr.nBl or typ != "object": - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}" + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}" else: - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = proc({pout}) {{.cdecl.}}" + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}) {{.cdecl.}}" else: - if i < nimState.data.len and nimState.data[i].name in ["identifier", "number_literal"]: + if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: var - flen = nimState.data[i].val - if nimState.data[i].name == "identifier": - flen = nimState.getIdentifier(flen, nskConst, nname) + flen = gState.data[i].val + if gState.data[i].name == "identifier": + flen = gState.getIdentifier(flen, nskConst, nname) - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]" + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]" else: if nname == typ: pragmas.add "incompleteStruct" let - pragma = nimState.getPragma(pragmas) - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = object" + pragma = gState.getPragma(pragmas) + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" else: - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = {getPtrType(tptr&typ)}" + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {getPtrType(tptr&typ)}" )) - proc pDupTypeCommon(nname: string, fend: int, nimState: NimState, isEnum=false) = - if nimState.gState.debug: - nimState.debugStr &= "\n# pDupTypeCommon()" + proc pDupTypeCommon(nname: string, fend: int, gState: State, isEnum=false) = + if gState.debug: + gState.debugStr &= "\n# pDupTypeCommon()" var - dname = nimState.data[^1].val - ndname = nimState.getIdentifier(dname, nskType) + dname = gState.data[^1].val + ndname = gState.getIdentifier(dname, nskType) dptr = if fend == 2: "ptr " @@ -265,21 +265,21 @@ proc initGrammar(): Grammar = if ndname.nBl and ndname != nname: if isEnum: - if nimState.addNewIdentifer(ndname): - nimState.enumStr &= &"{nimState.getComments(true)}\ntype {ndname}* = {dptr}{nname}" + if gState.addNewIdentifer(ndname): + gState.enumStr &= &"{gState.getComments(true)}\ntype {ndname}* = {dptr}{nname}" else: - if nimState.addNewIdentifer(ndname): + if gState.addNewIdentifer(ndname): let - pragma = nimState.getPragma(nimState.getImportc(dname, ndname), "bycopy") - nimState.typeStr &= - &"{nimState.getComments()}\n {ndname}*{pragma} = {dptr}{nname}" + pragma = gState.getPragma(gState.getImportc(dname, ndname), "bycopy") + gState.typeStr &= + &"{gState.getComments()}\n {ndname}*{pragma} = {dptr}{nname}" - proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# pStructCommon" + proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = + if gState.debug: + gState.debugStr &= "\n# pStructCommon" var - nname = nimState.getIdentifier(name, nskType) + nname = gState.getIdentifier(name, nskType) prefix = "" union = "" @@ -307,25 +307,25 @@ proc initGrammar(): Grammar = if not nname.nBl: let - override = nimState.getOverride(name, nskType) + override = gState.getOverride(name, nskType) if override.nBl: - nimState.typeStr &= &"{nimState.getComments()}\n{override}" - elif nimState.addNewIdentifer(nname): - if nimState.data.len == 1: - nimState.typeStr &= &"{nimState.getComments()}\n {nname}* {{.bycopy{union}.}} = object" + gState.typeStr &= &"{gState.getComments()}\n{override}" + elif gState.addNewIdentifer(nname): + if gState.data.len == 1: + gState.typeStr &= &"{gState.getComments()}\n {nname}* {{.bycopy{union}.}} = object" else: var pragmas: seq[string] = @[] - if nimState.includeHeader(): - pragmas.add nimState.getImportC(prefix & name, nname) + if gState.isIncludeHeader(): + pragmas.add gState.getImportC(prefix & name, nname) pragmas.add "bycopy" if union.nBl: pragmas.add "union" let - pragma = nimState.getPragma(pragmas) + pragma = gState.getPragma(pragmas) - nimState.typeStr &= &"{nimState.getComments()}\n {nname}*{pragma} = object" + gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" var i = fstart @@ -333,19 +333,19 @@ proc initGrammar(): Grammar = fptr = "" aptr = "" flen = "" - while i < nimState.data.len-fend: + while i < gState.data.len-fend: fptr = "" aptr = "" - if nimState.data[i].name == "field_declaration": + if gState.data[i].name == "field_declaration": i += 1 continue - if nimState.data[i].name notin ["field_identifier", "pointer_declarator", "array_pointer_declarator"]: - ftyp = nimState.getIdentifier(nimState.data[i].val, nskType, nname).getType() + if gState.data[i].name notin ["field_identifier", "pointer_declarator", "array_pointer_declarator"]: + ftyp = gState.getIdentifier(gState.data[i].val, nskType, nname).getType() i += 1 - while i < nimState.data.len-fend and "pointer" in nimState.data[i].name: - case nimState.data[i].name: + while i < gState.data.len-fend and "pointer" in gState.data[i].name: + case gState.data[i].name: of "pointer_declarator": fptr &= "ptr " i += 1 @@ -353,33 +353,33 @@ proc initGrammar(): Grammar = aptr &= "ptr " i += 1 - fname = nimState.getIdentifier(nimState.data[i].val, nskField, nname) + fname = gState.getIdentifier(gState.data[i].val, nskField, nname) - if i+1 < nimState.data.len-fend and nimState.data[i+1].name in gEnumVals: + if i+1 < gState.data.len-fend and gState.data[i+1].name in gEnumVals: # Struct field is an array where size is an expression var - flen = nimState.getNimExpression(nimState.data[i+1].val) + flen = gState.getNimExpression(gState.data[i+1].val) if "/" in flen: flen = &"({flen}).int" - nimState.typeStr &= &"{nimState.getComments()}\n {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]" + gState.typeStr &= &"{gState.getComments()}\n {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]" i += 2 - elif i+1 < nimState.data.len-fend and nimState.data[i+1].name == "bitfield_clause": + elif i+1 < gState.data.len-fend and gState.data[i+1].name == "bitfield_clause": let - size = nimState.data[i+1].val - nimState.typeStr &= &"{nimState.getComments()}\n {fname}* {{.bitsize: {size}.}} : {getPtrType(fptr&ftyp)} " + size = gState.data[i+1].val + gState.typeStr &= &"{gState.getComments()}\n {fname}* {{.bitsize: {size}.}} : {getPtrType(fptr&ftyp)} " i += 2 - elif i+1 < nimState.data.len-fend and nimState.data[i+1].name == "function_declarator": + elif i+1 < gState.data.len-fend and gState.data[i+1].name == "function_declarator": var pout, pname, ptyp, pptr = "" count = 1 i += 2 - while i < nimState.data.len-fend: - if nimState.data[i].name == "function_declarator": + while i < gState.data.len-fend: + if gState.data[i].name == "function_declarator": i += 1 continue - if nimState.data[i].name == "field_declaration": + if gState.data[i].name == "field_declaration": break funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) @@ -387,20 +387,20 @@ proc initGrammar(): Grammar = if pout.nBl and pout[^2 .. ^1] == ", ": pout = pout[0 .. ^3] if fptr.nBl or ftyp != "object": - nimState.typeStr &= &"{nimState.getComments()}\n {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}" + gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}" else: - nimState.typeStr &= &"{nimState.getComments()}\n {fname}*: proc({pout}) {{.cdecl.}}" + gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}) {{.cdecl.}}" i += 1 else: if ftyp == "object": - nimState.typeStr &= &"{nimState.getComments()}\n {fname}*: pointer" + gState.typeStr &= &"{gState.getComments()}\n {fname}*: pointer" else: - nimState.typeStr &= &"{nimState.getComments()}\n {fname}*: {getPtrType(fptr&ftyp)}" + gState.typeStr &= &"{gState.getComments()}\n {fname}*: {getPtrType(fptr&ftyp)}" i += 1 if node.tsNodeType() == "type_definition" and - nimState.data[^1].name == "type_identifier" and nimState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, nimState, false) + gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: + pDupTypeCommon(nname, fend, gState, false) let fieldGrammar = &""" @@ -451,11 +451,11 @@ proc initGrammar(): Grammar = {fieldListGrammar} ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# struct X {}" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# struct X {}" - pStructCommon(ast, node, nimState.data[0].val, 1, 1, nimState) + pStructCommon(ast, node, gState.data[0].val, 1, 1, gState) )) # typedef struct X {} @@ -474,69 +474,69 @@ proc initGrammar(): Grammar = ) ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# typedef struct X {}" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# typedef struct X {}" var fstart = 0 fend = 1 - if nimState.data[^2].name == "pointer_declarator": + if gState.data[^2].name == "pointer_declarator": fend = 2 - if nimState.data.len > 1 and - nimState.data[0].name == "type_identifier" and - nimState.data[1].name notin ["field_identifier", "pointer_declarator"]: + if gState.data.len > 1 and + gState.data[0].name == "type_identifier" and + gState.data[1].name notin ["field_identifier", "pointer_declarator"]: fstart = 1 - pStructCommon(ast, node, nimState.data[0].val, fstart, fend, nimState) + pStructCommon(ast, node, gState.data[0].val, fstart, fend, gState) else: - pStructCommon(ast, node, nimState.data[^1].val, fstart, fend, nimState) + pStructCommon(ast, node, gState.data[^1].val, fstart, fend, gState) )) - proc pEnumCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# pEnumCommon()" + proc pEnumCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = + if gState.debug: + gState.debugStr &= "\n# pEnumCommon()" let nname = if name.Bl: - getUniqueIdentifier(nimState, "Enum") + getUniqueIdentifier(gState, "Enum") else: - nimState.getIdentifier(name, nskType) + gState.getIdentifier(name, nskType) - if nname.nBl and nimState.addNewIdentifer(nname): - nimState.enumStr &= &"{nimState.getComments(true)}\ndefineEnum({nname})" + if nname.nBl and gState.addNewIdentifer(nname): + gState.enumStr &= &"{gState.getComments(true)}\ndefineEnum({nname})" var i = fstart count = 0 - while i < nimState.data.len-fend: - if nimState.data[i].name == "enumerator": + while i < gState.data.len-fend: + if gState.data[i].name == "enumerator": i += 1 continue let - fname = nimState.getIdentifier(nimState.data[i].val, nskEnumField) + fname = gState.getIdentifier(gState.data[i].val, nskEnumField) - if i+1 < nimState.data.len-fend and - nimState.data[i+1].name in gEnumVals: - if fname.nBl and nimState.addNewIdentifer(fname): - nimState.constStr &= &"{nimState.getComments()}\n {fname}* = ({nimState.getNimExpression(nimState.data[i+1].val)}).{nname}" + if i+1 < gState.data.len-fend and + gState.data[i+1].name in gEnumVals: + if fname.nBl and gState.addNewIdentifer(fname): + gState.constStr &= &"{gState.getComments()}\n {fname}* = ({gState.getNimExpression(gState.data[i+1].val)}).{nname}" try: - count = nimState.data[i+1].val.parseInt() + 1 + count = gState.data[i+1].val.parseInt() + 1 except: count += 1 i += 2 else: - if fname.nBl and nimState.addNewIdentifer(fname): - nimState.constStr &= &"{nimState.getComments()}\n {fname}* = {count}.{nname}" + if fname.nBl and gState.addNewIdentifer(fname): + gState.constStr &= &"{gState.getComments()}\n {fname}* = {count}.{nname}" i += 1 count += 1 if node.tsNodeType() == "type_definition" and - nimState.data[^1].name == "type_identifier" and nimState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, nimState, true) + gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: + pDupTypeCommon(nname, fend, gState, true) # enum X {} result.add((""" @@ -550,19 +550,19 @@ proc initGrammar(): Grammar = ) ) """ % gEnumVals.join("|"), - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# enum X {}" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# enum X {}" var name = "" offset = 0 - if nimState.data[0].name == "type_identifier": - name = nimState.data[0].val + if gState.data[0].name == "type_identifier": + name = gState.data[0].val offset = 1 - pEnumCommon(ast, node, name, offset, 0, nimState) + pEnumCommon(ast, node, name, offset, 0, gState) )) # typedef enum {} X @@ -578,23 +578,23 @@ proc initGrammar(): Grammar = ) ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# typedef enum {}" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# typedef enum {}" var fstart = 0 fend = 1 - if nimState.data[^2].name == "pointer_declarator": + if gState.data[^2].name == "pointer_declarator": fend = 2 - if nimState.data[0].name == "type_identifier": + if gState.data[0].name == "type_identifier": fstart = 1 - pEnumCommon(ast, node, nimState.data[0].val, fstart, fend, nimState) + pEnumCommon(ast, node, gState.data[0].val, fstart, fend, gState) else: - pEnumCommon(ast, node, nimState.data[^1].val, fstart, fend, nimState) + pEnumCommon(ast, node, gState.data[^1].val, fstart, fend, gState) )) # typ function(typ param1, ...) @@ -611,39 +611,39 @@ proc initGrammar(): Grammar = {funcGrammar} ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = - if nimState.gState.debug: - nimState.debugStr &= "\n# typ function" + proc (ast: ref Ast, node: TSNode, gState: State) = + if gState.debug: + gState.debugStr &= "\n# typ function" var fptr = "" i = 1 - while i < nimState.data.len: - if nimState.data[i].name == "function_declarator": + while i < gState.data.len: + if gState.data[i].name == "function_declarator": i += 1 continue fptr = "" - while i < nimState.data.len and nimState.data[i].name == "pointer_declarator": + while i < gState.data.len and gState.data[i].name == "pointer_declarator": fptr &= "ptr " i += 1 var - fname = nimState.data[i].val - fnname = nimState.getIdentifier(fname, nskProc) + fname = gState.data[i].val + fnname = gState.getIdentifier(fname, nskProc) pout, pname, ptyp, pptr = "" count = 1 flen = "" fVar = false i += 1 - if i < nimState.data.len and nimState.data[i].name == "pointer_declarator": + if i < gState.data.len and gState.data[i].name == "pointer_declarator": fVar = true i += 1 - while i < nimState.data.len: - if nimState.data[i].name == "function_declarator": + while i < gState.data.len: + if gState.data[i].name == "function_declarator": break funcParamCommon(fnname, pname, ptyp, pptr, pout, count, i, flen) @@ -653,24 +653,24 @@ proc initGrammar(): Grammar = if not fnname.nBl: let - override = nimState.getOverride(fname, nskProc) + override = gState.getOverride(fname, nskProc) if override.nBl: - nimState.typeStr &= &"{nimState.getComments()}\n{override}" - elif nimState.addNewIdentifer(fnname): + gState.typeStr &= &"{gState.getComments()}\n{override}" + elif gState.addNewIdentifer(fnname): let - ftyp = nimState.getIdentifier(nimState.data[0].val, nskType, fnname).getType() - pragma = nimState.getPragma(nimState.getImportC(fname, fnname), "cdecl") + ftyp = gState.getIdentifier(gState.data[0].val, nskType, fnname).getType() + pragma = gState.getPragma(gState.getImportC(fname, fnname), "cdecl") if fptr.nBl or ftyp != "object": if fVar: - nimState.procStr &= &"{nimState.getComments(true)}\nvar {fnname}*: proc ({pout}): {getPtrType(fptr&ftyp)}{{.cdecl.}}" + gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}): {getPtrType(fptr&ftyp)}{{.cdecl.}}" else: - nimState.procStr &= &"{nimState.getComments(true)}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}" + gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}" else: if fVar: - nimState.procStr &= &"{nimState.getComments(true)}\nvar {fnname}*: proc ({pout}){{.cdecl.}}" + gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}){{.cdecl.}}" else: - nimState.procStr &= &"{nimState.getComments(true)}\nproc {fnname}*({pout}){pragma}" + gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}){pragma}" )) # // comment @@ -678,15 +678,15 @@ proc initGrammar(): Grammar = (comment ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = + proc (ast: ref Ast, node: TSNode, gState: State) = let - cmt = $nimState.getNodeVal(node) + cmt = $gState.getNodeVal(node) for line in cmt.splitLines(): let line = line.multiReplace([("//", ""), ("/*", ""), ("*/", "")]) - nimState.commentStr &= &"\n # {line.strip(leading=false)}" + gState.commentStr &= &"\n # {line.strip(leading=false)}" )) # // unknown @@ -695,37 +695,37 @@ proc initGrammar(): Grammar = (^.*) ) """, - proc (ast: ref Ast, node: TSNode, nimState: NimState) = + proc (ast: ref Ast, node: TSNode, gState: State) = var done = false - for i in nimState.data: + for i in gState.data: case $node.tsNodeType() of "declaration": if i.name == "identifier": let - override = nimState.getOverride(i.val, nskProc) + override = gState.getOverride(i.val, nskProc) if override.nBl: - nimState.procStr &= &"{nimState.getComments(true)}\n{override}" + gState.procStr &= &"{gState.getComments(true)}\n{override}" done = true break else: - nimState.procStr &= &"{nimState.getComments(true)}\n# Declaration '{i.val}' skipped" + gState.procStr &= &"{gState.getComments(true)}\n# Declaration '{i.val}' skipped" else: if i.name == "type_identifier": let - override = nimState.getOverride(i.val, nskType) + override = gState.getOverride(i.val, nskType) if override.nBl: - nimState.typeStr &= &"{nimState.getComments()}\n{override}" + gState.typeStr &= &"{gState.getComments()}\n{override}" done = true break else: - nimState.typeStr &= &"{nimState.getComments()}\n # Type '{i.val}' skipped" + gState.typeStr &= &"{gState.getComments()}\n # Type '{i.val}' skipped" - if nimState.gState.debug and not done: - nimState.skipStr &= &"\n{nimState.getNodeVal(node)}" + if gState.debug and not done: + gState.skipStr &= &"\n{gState.getNodeVal(node)}" )) proc initRegex(ast: ref Ast) = From f2870d8ba6fe9249f26d681b3f01cfb8b121d362 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 21 Apr 2020 23:37:37 -0500 Subject: [PATCH 055/255] Fix multi-header processing, perf improvements --- nimterop/ast.nim | 7 ++++--- nimterop/ast2.nim | 24 +++++++++++++++--------- nimterop/getters.nim | 23 +++++++++++++---------- nimterop/toast.nim | 23 +++++++++++++++++------ 4 files changed, 49 insertions(+), 28 deletions(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 6b21bc7..1f1cac6 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -202,12 +202,11 @@ proc searchAst(root: TSNode, astTable: AstTable, gState: State) = if node == root: break -proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = +proc parseNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = + # Generate Nim from tree-sitter AST root node var fp = fullpath.replace("\\", "/") - gState.identifiers = newTable[string, string]() - gState.currentHeader = getCurrentHeader(fullpath) gState.impShort = gState.currentHeader.replace("header", "imp") gState.sourceFile = fullpath @@ -217,6 +216,8 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable root.searchAst(astTable, gState) +proc printNim*(gState: State) = + # Print Nim generated by parseNim() if gState.enumStr.nBl: gecho &"{gState.enumStr}\n" diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 9e8f582..0e7a7bb 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1777,20 +1777,13 @@ proc printNimHeader*(gState: State) = import nimterop/types """ % [$now(), getAppFilename(), commandLineParams().join(" ")] -proc printNim*(gState: State, fullpath: string, root: TSNode) = - # Generate Nim from tree-sitter AST root node - let - fp = fullpath.replace("\\", "/") +proc initNim*(gState: State) = + # Initialize for parseNim() one time # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() gState.identifierNodes = newTable[string, PNode]() - # toast objects - gState.currentHeader = getCurrentHeader(fullpath) - gState.impShort = gState.currentHeader.replace("header", "imp") - gState.sourceFile = fullpath - # Nim compiler objects gState.identCache = newIdentCache() gState.config = newConfigRef() @@ -1804,12 +1797,25 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = gState.typeSection = newNode(nkTypeSection) gState.varSection = newNode(nkVarSection) +proc parseNim*(gState: State, fullpath: string, root: TSNode) = + # Generate Nim from tree-sitter AST root node + let + fp = fullpath.replace("\\", "/") + + # toast objects + gState.currentHeader = getCurrentHeader(fullpath) + gState.impShort = gState.currentHeader.replace("header", "imp") + gState.sourceFile = fullpath + # Setup pragmas gState.setupPragmas(root, fp) # Search root node and render Nim gState.searchTree(root) +proc printNim*(gState: State) = + # Print Nim generated by parseNim() + # Add any unused cOverride symbols to output gState.addAllOverrideFinal() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 1cdd2ff..88786ce 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -351,13 +351,10 @@ proc inChildren*(node: TSNode, ntype: string): bool = proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = # Get line number and column info for node - result.line = 1 - result.col = 1 - for i in 0 .. node.tsNodeStartByte().int-1: - if gState.code[i] == '\n': - result.col = 0 - result.line += 1 - result.col += 1 + let + point = node.tsNodeStartPoint() + result.line = point.row.int + 1 + result.col = point.column.int + 1 proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = for i in 0 ..< node.len: @@ -542,6 +539,14 @@ proc getPreprocessor*(gState: State, fullpath: string): string = for def in gState.defines: cmd &= &"-D{def} " + # Remove gcc special calls + if defined(posix): + cmd &= "-D__attribute__\\(x\\)= " + else: + cmd &= "-D__attribute__(x)= " + + cmd &= "-D__restrict= " + cmd &= &"{fullpath.sanitizePath}" # Include content only from file @@ -570,9 +575,7 @@ proc getPreprocessor*(gState: State, fullpath: string): string = if "#undef" in line: continue rdata.add line - return rdata.join("\n"). - replace("__restrict", ""). - replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1") + return rdata.join("\n") converter toString*(kind: Kind): string = return case kind: diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 3f886e1..7dad591 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -1,4 +1,4 @@ -import os, osproc, strformat, strutils, tables, times +import os, osproc, segfaults, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] @@ -40,9 +40,9 @@ proc process(gState: State, path: string, astTable: AstTable) = gecho gState.printLisp(root) elif gState.pnim: if Feature.ast2 in gState.feature: - ast2.printNim(gState, path, root) + ast2.parseNim(gState, path, root) else: - ast.printNim(gState, path, root, astTable) + ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code @@ -137,17 +137,28 @@ proc main( # Process grammar into AST let - astTable = parseGrammar() + astTable = + if Feature.ast2 notin gState.feature: + parseGrammar() + else: + nil if pgrammar: - # Print AST of grammar - gState.printGrammar(astTable) + if Feature.ast2 notin gState.feature: + # Print AST of grammar + gState.printGrammar(astTable) elif source.nBl: # Print source after preprocess or Nim output if gState.pnim: gState.printNimHeader() + gState.initNim() for src in source: gState.process(src.expandSymlinkAbs(), astTable) + if gState.pnim: + if Feature.ast2 in gState.feature: + ast2.printNim(gState) + else: + ast.printNim(gState) # Close outputFile if outputFile.len != 0: From 00b66bf92c8b566f6f0bd27595ea16a13acfa549 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 22 Apr 2020 09:25:23 -0500 Subject: [PATCH 056/255] Change FILE* to File --- nimterop/ast2.nim | 2 ++ nimterop/grammar.nim | 2 ++ tests/config.nims.disabled | 7 ------- tests/nim.cfg | 3 +-- 4 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 tests/config.nims.disabled diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 0e7a7bb..ce45f1d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -14,6 +14,8 @@ proc getPtrType*(str: string): string = "cstring" of "object": "pointer" + of "FILE": + "File" else: str diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index 0d804d0..eaa337a 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -17,6 +17,8 @@ proc getPtrType(str: string): string = "pointer" of "ptr ptr object": "ptr pointer" + of "ptr FILE": + "File" else: str diff --git a/tests/config.nims.disabled b/tests/config.nims.disabled deleted file mode 100644 index 06a791e..0000000 --- a/tests/config.nims.disabled +++ /dev/null @@ -1,7 +0,0 @@ -#[ -pending https://github.com/nim-lang/Nim/pull/10530 -note: nimble init installs something like this (maybe without src in this case) -switch("path", "$projectDir/../src") -but it doesn't seem robust in case tests have subdirs, so, changing to ../ seems better -]# -switch("path", "..") diff --git a/tests/nim.cfg b/tests/nim.cfg index 3936716..5bb1fab 100644 --- a/tests/nim.cfg +++ b/tests/nim.cfg @@ -1,2 +1 @@ -# TODO: pending https://github.com/nimterop/nimterop/issues/110 remove in favor of config.nims ---path:".." +--path:".." \ No newline at end of file From 79526221b7cd488043f545a903f65da736eea2c7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 22 Apr 2020 11:28:16 -0500 Subject: [PATCH 057/255] Avoid building toast twice in CI --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b66a7e..d4e9bde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,6 @@ install: - source travis.sh script: - - nimble --verbose install -y - - nimble --verbose test - - nimble --verbose --nimbleDir:`pwd`/build/fakenimble install nimterop -y + - nimble develop -y + - nimble test + - nimble --verbose --nimbleDir:`pwd`/build/fakenimble install nimterop@#head -y From da6a4e30ab9c88748e54e4cf0bdae248c8750cdd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 22 Apr 2020 11:37:45 -0500 Subject: [PATCH 058/255] Remove segfaults --- nimterop/toast.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 7dad591..ab44970 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -1,4 +1,4 @@ -import os, osproc, segfaults, strformat, strutils, tables, times +import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] From f17b958f47686f41a20045ea089295841a6845d6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 22 Apr 2020 12:01:20 -0500 Subject: [PATCH 059/255] markAndSweep, removeStatic --- config.nims | 4 ++-- nimterop/getters.nim | 16 +--------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/config.nims b/config.nims index 6b475de..f5f7d21 100644 --- a/config.nims +++ b/config.nims @@ -5,8 +5,8 @@ else: switch("gcc.linkerexe", "g++") # Workaround for NilAccessError crash on Windows #98 -when defined(Windows): - switch("gc", "markAndSweep") +# Could also help for OSX/Linux crash +switch("gc", "markAndSweep") # Retain stackTrace for clear errors switch("stackTrace", "on") diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 88786ce..2d8d9bb 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -510,20 +510,6 @@ proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) -proc removeStatic(content: string): string = - ## Replace static function bodies with a semicolon and commented - ## out body - return content.replace( - re"(?msU)static inline ([^)]+\))([^}]+\})", - proc (m: RegexMatch, s: string): string = - let funcDecl = s[m.group(0)[0]] - let body = s[m.group(1)[0]].strip() - result = "" - - result.add("$#;" % [funcDecl]) - result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) - ) - proc getPreprocessor*(gState: State, fullpath: string): string = var cmts = if gState.nocomments: "" else: "-CC" @@ -731,7 +717,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = pdll = sourcePath.dll if not fileExists(pdll) or sourcePath.getLastModificationTime() > pdll.getLastModificationTime(): - discard execAction(&"{gState.nim.sanitizePath} c --app:lib {sourcePath.sanitizePath}") + discard execAction(&"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {sourcePath.sanitizePath}") doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath let lib = loadLib(pdll) From c1f46680f77d077bfc7abebea8b289e9b8bac043 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 22 Apr 2020 22:11:30 -0500 Subject: [PATCH 060/255] Add cImport for multiple headers --- nimterop.nimble | 3 ++ nimterop/cimport.nim | 91 ++++++++++++++++++++++++++------------------ tests/rsa.nim | 42 ++++++++++++++++++++ 3 files changed, 99 insertions(+), 37 deletions(-) create mode 100644 tests/rsa.nim diff --git a/nimterop.nimble b/nimterop.nimble index 37c395d..9cb79b1 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -51,6 +51,9 @@ task test, "Test": execTest "tests/tpcre.nim" execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" + execTest "tests/rsa.nim" + execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"" + # Platform specific tests when defined(Windows): execTest "tests/tmath.nim" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 52d1892..3e58bc8 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -91,6 +91,11 @@ proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() +proc getCacheValue(fullpaths: seq[string]): string = + if not gStateCT.nocache: + for fullpath in fullpaths: + result &= getCacheValue(fullpath) + proc getToastError(output: string): string = # Filter out preprocessor errors for line in output.splitLines(): @@ -121,7 +126,7 @@ proc getNimCheckError(output: string): tuple[tmpFile, errors: string] = result.errors = "\n\n" & check -proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", +proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "", mode = "c", flags = "", noNimout = false): string = var ret = 0 @@ -158,11 +163,12 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", if gStateCT.pluginSourcePath.nBl: cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}" - cmd.add &" {fullpath.sanitizePath}" + for fullpath in fullpaths: + cmd.add &" {fullpath.sanitizePath}" # see https://github.com/nimterop/nimterop/issues/69 (result, ret) = execAction(cmd, die = false, cache = (not gStateCT.nocache), - cacheKey = getCacheValue(fullpath)) + cacheKey = getCacheValue(fullpaths)) doAssert ret == 0, getToastError(result) macro cOverride*(body): untyped = @@ -555,6 +561,46 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = if gStateCT.debug: echo result.repr +macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", + mode: static string = "c", flags: static string = ""): untyped = + ## Import multiple headers in one shot + ## + ## This macro is preferable over multiple individual `cImport()` calls, especially + ## when the headers might `#include` the same headers and result in duplicate symbols. + result = newNimNode(nnkStmtList) + + var + fullpaths: seq[string] + + for filename in filenames: + fullpaths.add findPath(filename) + + # In case cOverride called after cPlugin + if gStateCT.pluginSourcePath.Bl: + cPluginHelper(gStateCT.pluginSource) + + echo "# Importing " & fullpaths.join(", ").sanitizePath + + let + output = getToast(fullpaths, recurse, dynlib, mode, flags) + + # Reset plugin and overrides for next cImport + if gStateCT.overrides.nBl: + gStateCT.pluginSourcePath = "" + gStateCT.overrides = "" + + if gStateCT.debug: + echo output + + try: + let body = parseStmt(output) + + result.add body + except: + let + (tmpFile, errors) = getNimCheckError(output) + doAssert false, errors & "\n\nNimterop codegen limitation or error - review 'nim check' output above generated for " & tmpFile + macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = ## Import all supported definitions from specified header file. Generated @@ -590,8 +636,8 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## with `cCompile() `_, or the ## `{.passL.}` pragma can be used to specify the static lib to link. ## - ## `mode` is purely for forward compatibility when toast adds C++ support. It can - ## be ignored for the foreseeable future. + ## `mode` selects the preprocessor and tree-sitter parser to be used to process + ## the header. ## ## `flags` can be used to pass any other command line arguments to `toast`. A ## good example would be `--prefix` and `--suffix` which strip leading and @@ -600,37 +646,8 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## `cImport()` consumes and resets preceding `cOverride()` calls. `cPlugin()` ## is retained for the next `cImport()` call unless a new `cPlugin()` call is ## defined. - - result = newNimNode(nnkStmtList) - - let - fullpath = findPath(filename) - - # In case cOverride called after cPlugin - if gStateCT.pluginSourcePath.Bl: - cPluginHelper(gStateCT.pluginSource) - - echo "# Importing " & fullpath.sanitizePath - - let - output = getToast(fullpath, recurse, dynlib, mode, flags) - - # Reset plugin and overrides for next cImport - if gStateCT.overrides.nBl: - gStateCT.pluginSourcePath = "" - gStateCT.overrides = "" - - if gStateCT.debug: - echo output - - try: - let body = parseStmt(output) - - result.add body - except: - let - (tmpFile, errors) = getNimCheckError(output) - doAssert false, errors & "\n\nNimterop codegen limitation or error - review 'nim check' output above generated for " & tmpFile + return quote do: + cImport(@[`filename`], bool(`recurse`), `dynlib`, `mode`, `flags`) macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = @@ -661,7 +678,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: echo "# Importing " & fullpath & " with c2nim" let - output = getToast(fullpath, recurse, dynlib, noNimout = true) + output = getToast(@[fullpath], recurse, dynlib, noNimout = true) hash = output.hash().abs() hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" npath = hpath[0 .. hpath.rfind('.')] & "nim" diff --git a/tests/rsa.nim b/tests/rsa.nim new file mode 100644 index 0000000..638c490 --- /dev/null +++ b/tests/rsa.nim @@ -0,0 +1,42 @@ +import os +import strutils + +import nimterop/[build, cimport] + +setDefines(@["cryptoStd"]) +getHeader("openssl/crypto.h") + +const + basePath = cryptoPath.parentDir + FLAGS {.strdefine.} = "" + +cPlugin: + import strutils + + proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = + sym.name = sym.name.strip(chars = {'_'}).replace("__", "_") + + if sym.name in [ + "AES_ENCRYPT", "AES_DECRYPT", + "BIO_CTRL_PENDING", "BIO_CTRL_WPENDING", + "BN_F_BNRAND", "BN_F_BNRAND_RANGE", + "CRYPTO_THREADID", + "EVP_CIPHER", + "OPENSSL_VERSION", + "PKCS7_ENCRYPT", "PKCS7_STREAM", + "SSL_TXT_ADH", "SSL_TXT_AECDH", "SSL_TXT_kECDHE" + ]: + sym.name = "C_" & sym.name + +cOverride: + proc OPENSSL_die*(assertion: cstring; file: cstring; line: cint) {.importc.} + +cImport(@[ + basePath / "rsa.h", + basePath / "err.h", +], recurse = true, flags = "-f:ast2 -s " & FLAGS) + +{.passL: cryptoLPath.} + +OpensslInit() +echo $OPENSSL_VERSION_TEXT \ No newline at end of file From 43dd43e3183178e71abd3319290c566cb4dd80a9 Mon Sep 17 00:00:00 2001 From: Travis CI User Date: Thu, 23 Apr 2020 14:34:41 +0000 Subject: [PATCH 061/255] Fix rsa test --- nimterop.nimble | 5 +++-- tests/rsa.nim | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 9cb79b1..a6fba1e 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -51,8 +51,9 @@ task test, "Test": execTest "tests/tpcre.nim" execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/rsa.nim" - execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"" + when defined(Linux): + execTest "tests/rsa.nim" + execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"" # Platform specific tests when defined(Windows): diff --git a/tests/rsa.nim b/tests/rsa.nim index 638c490..e7bde89 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -10,6 +10,9 @@ const basePath = cryptoPath.parentDir FLAGS {.strdefine.} = "" +static: + cSkipSymbol(@["ERR_load_crypto_strings", "OpenSSLDie"]) + cPlugin: import strutils @@ -20,10 +23,11 @@ cPlugin: "AES_ENCRYPT", "AES_DECRYPT", "BIO_CTRL_PENDING", "BIO_CTRL_WPENDING", "BN_F_BNRAND", "BN_F_BNRAND_RANGE", - "CRYPTO_THREADID", + "CRYPTO_LOCK", "CRYPTO_NUM_LOCKS", "CRYPTO_THREADID", "EVP_CIPHER", "OPENSSL_VERSION", "PKCS7_ENCRYPT", "PKCS7_STREAM", + "SSLEAY_VERSION", "SSL_TXT_ADH", "SSL_TXT_AECDH", "SSL_TXT_kECDHE" ]: sym.name = "C_" & sym.name @@ -39,4 +43,4 @@ cImport(@[ {.passL: cryptoLPath.} OpensslInit() -echo $OPENSSL_VERSION_TEXT \ No newline at end of file +echo $OPENSSL_VERSION_TEXT From 80018f43cf35214e95c4fa777f65bb756be1db24 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 14 Apr 2020 20:38:35 -0600 Subject: [PATCH 062/255] WIP Ast printing --- nimterop/ast2.nim | 30 +++++++++++++++++++++++++++++- nimterop/getters.nim | 21 +++++++++++++++------ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index ce45f1d..43b6b35 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,10 +4,35 @@ import regex import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] -import "."/treesitter/api +import "."/treesitter/[api, c, cpp] import "."/[globals, getters] +proc getCCodeAst*(gState: State, code: string): string = + var parser = tsParserNew() + var code = code + + defer: + parser.tsParserDelete() + + + doAssert code.nBl, "Empty code" + if gState.mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif gState.mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {gState.mode}" + + var + tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) + root = tree.tsTreeRootNode() + + defer: + tree.tsTreeDelete() + + return code.printLisp(root) + proc getPtrType*(str: string): string = result = case str: of "cchar": @@ -59,6 +84,9 @@ proc getLit*(gState: State, str: string, expression = false): PNode = result = newStrNode(nkStrLit, str[1 .. ^2]) else: + decho "Macro AST:" + decho str + decho nimState.gState.getCCodeAst(str) let str = if expression: gState.getNimExpression(str) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 2d8d9bb..4ffb5d7 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -228,9 +228,12 @@ proc getName*(node: TSNode): string {.inline.} = if not node.isNil: return $node.tsNodeType() -proc getNodeVal*(gState: State, node: TSNode): string = +proc getNodeVal*(code: var string, node: TSNode): string = if not node.isNil: - return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + +proc getNodeVal*(gState: State, node: TSNode): string = + gState.code.getNodeVal(node) proc getAtom*(node: TSNode): TSNode = if not node.isNil: @@ -349,13 +352,16 @@ proc inChildren*(node: TSNode, ntype: string): bool = result = true break -proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = +proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = # Get line number and column info for node let point = node.tsNodeStartPoint() result.line = point.row.int + 1 result.col = point.column.int + 1 +proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getLineCol(gState.code, node) + proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = for i in 0 ..< node.len: if node.getName() != "comment": @@ -374,7 +380,7 @@ proc getPxName*(node: TSNode, offset: int): string = if count == offset and not np.isNil: return np.getName() -proc printLisp*(gState: State, root: TSNode): string = +proc printLisp*(code: var string, root: TSNode): string = var node = root nextnode: TSNode @@ -384,10 +390,10 @@ proc printLisp*(gState: State, root: TSNode): string = if not node.isNil and depth > -1: result &= spaces(depth) let - (line, col) = gState.getLineCol(node) + (line, col) = code.getLineCol(node) result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" let - val = gState.getNodeVal(node) + val = code.getNodeVal(node) if "\n" notin val and " " notin val: result &= &" \"{val}\"" else: @@ -419,6 +425,9 @@ proc printLisp*(gState: State, root: TSNode): string = if node == root: break +proc printLisp*(gState: State, root: TSNode): string = + printLisp(gState.code, root) + proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") From 879c7e3c784b3debce068a93e7f9ee116eb7bc21 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 16 Apr 2020 20:49:06 -0600 Subject: [PATCH 063/255] Add preliminary expression parsing --- nimterop/ast2.nim | 77 ++----------- nimterop/exprparser.nim | 245 ++++++++++++++++++++++++++++++++++++++++ nimterop/getters.nim | 2 +- 3 files changed, 253 insertions(+), 71 deletions(-) create mode 100644 nimterop/exprparser.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 43b6b35..04bb8eb 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,37 +1,10 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import regex - import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] -import "."/treesitter/[api, c, cpp] +import "."/treesitter/api -import "."/[globals, getters] - -proc getCCodeAst*(gState: State, code: string): string = - var parser = tsParserNew() - var code = code - - defer: - parser.tsParserDelete() - - - doAssert code.nBl, "Empty code" - if gState.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif gState.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {gState.mode}" - - var - tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) - root = tree.tsTreeRootNode() - - defer: - tree.tsTreeDelete() - - return code.printLisp(root) +import "."/[globals, getters, exprparser] proc getPtrType*(str: string): string = result = case str: @@ -59,42 +32,8 @@ proc parseString(gState: State, str: string): PNode = except: decho getCurrentExceptionMsg() -proc getLit*(gState: State, str: string, expression = false): PNode = - # Used to convert #define literals into const and expressions - # in array sizes - # - # `expression` is true when `str` should be converted into a Nim expression - let - str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - - if str.contains(re"^[\-]?[\d]+$"): # decimal - result = newIntNode(nkIntLit, parseInt(str)) - - elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float - result = newFloatNode(nkFloatLit, parseFloat(str)) - - elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - result = gState.parseString(str) - - elif str.contains(re"^'[[:ascii:]]'$"): # char - result = newNode(nkCharLit) - result.intVal = str[1].int64 - - elif str.contains(re"""^"[[:ascii:]]+"$"""): # char * - result = newStrNode(nkStrLit, str[1 .. ^2]) - - else: - decho "Macro AST:" - decho str - decho nimState.gState.getCCodeAst(str) - let - str = - if expression: gState.getNimExpression(str) - else: str - result = gState.parseString(str) - - if result.isNil: - result = newNode(nkNilLit) +proc getLit*(nimState: NimState, str: string, expression = false): PNode = + result = nimState.codeToNode(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -181,11 +120,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank - result = gState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind in {nkCharLit .. nkStrLit} or - (valident.kind == nkStmtList and valident.len > 0 and - valident[0].kind in {nkCharLit .. nkStrLit}): - if gState.addNewIdentifer(name): + result = nimState.getOverrideOrSkip(node, origname, nskConst) + elif valident.kind != nkNilLit: + if nimState.addNewIdentifer(name): # const X* = Y # # nkConstDef( diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim new file mode 100644 index 0000000..a4be5d0 --- /dev/null +++ b/nimterop/exprparser.nim @@ -0,0 +1,245 @@ +import strformat, strutils, macros + +import regex + +import compiler/[ast, renderer] + +import "."/treesitter/[api, c, cpp] + +import "."/[globals, getters] + +type + ExprParser* = ref object + state*: NimState + code*: string + +proc newExprParser*(state: NimState, code: string): ExprParser = + ExprParser(state: state, code: code) + +template decho(msg: varargs[string, `$`]) = + if exprParser.state.gState.debug: + let nimState {.inject.} = exprParser.state + necho "# " & join(msg, "") + +template val*(node: TSNode): string = + exprParser.code.getNodeVal(node) + +proc mode*(exprParser: ExprParser): string = + exprParser.state.gState.mode + +template withCodeAst(exprParser: ExprParser, body: untyped): untyped = + var parser = tsParserNew() + defer: + parser.tsParserDelete() + + doAssert exprParser.code.nBl, "Empty code" + if exprParser.mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif exprParser.mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {exprParser.mode}" + + var + tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32) + root {.inject.} = tree.tsTreeRootNode() + + body + + defer: + tree.tsTreeDelete() + + +proc getNumNode(number, suffix: string): PNode {.inline.} = + result = newNode(nkNilLit) + if number.contains("."): + let floatSuffix = number[result.len-1] + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + discard + return + + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) + + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) + +proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkNilLit) + let nodeVal = node.val + + var match: RegexMatch + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)" + let found = nodeVal.find(reg, match) + if found: + let + prefix = if match.group(0).len > 0: nodeVal[match.group(0)[0]] else: "" + number = nodeVal[match.group(1)[0]] + suffix = nodeVal[match.group(2)[0]] + + result = getNumNode(number, suffix) + + if result.kind != nkNilLit and prefix == "-": + result = nkPrefix.newTree( + exprParser.state.getIdent("-"), + result + ) + +proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkCharLit) + result.intVal = node.val[1].int64 + +proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = + let nodeVal = node.val + result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) + +proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode + +proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkInfix) + let + left = node[0] + right = node[1] + var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + + case shiftSym + of "<<": + result.add exprParser.state.getIdent("shl") + of ">>": + result.add exprParser.state.getIdent("shr") + else: + discard + + result.add exprParser.processTSNode(left) + result.add exprParser.processTSNode(right) + +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkPar) + for i in 0 ..< node.len(): + result.add(exprParser.processTSNode(node[i])) + +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + decho "LOG SYM: ", binarySym + + case binarySym + of "!": + nimSym = "not" + else: + return newNode(nkNilLit) + + decho "LOG CHILD: ", child.val, ", nim: ", nimSym + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child) + ) + +proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = + if node.len() > 1: + result = newNode(nkInfix) + let left = node[0] + let right = node[1] + var nimSym = "" + + var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + decho "# BIN SYM: ", binarySym + + case binarySym + of "|", "||": + nimSym = "or" + of "&", "&&": + nimSym = "and" + of "^": + nimSym = "xor" + else: + return newNode(nkNilLit) + + result.add exprParser.state.getIdent(nimSym) + result.add exprParser.processTSNode(left) + result.add exprParser.processTSNode(right) + + elif node.len() == 1: + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + decho "# BIN SYM: ", binarySym + + case binarySym + of "~": + nimSym = "not" + else: + return newNode(nkNilLit) + + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child) + ) + +proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkNilLit) + decho "# NODE: ", node.getName(), ", VAL: ", node.val + case node.getName() + of "number_literal": + result = exprParser.processNumberLiteral(node) + of "string_literal": + result = exprParser.processStringLiteral(node) + of "char_literal": + result = exprParser.processCharacterLiteral(node) + of "expression_statement", "ERROR", "translation_unit": + # This may be wrong. What can be in an expression? + result = exprParser.processTSNode(node[0]) + of "parenthesized_expression": + result = exprParser.processParenthesizedExpr(node) + of "bitwise_expression": + result = exprParser.processBitwiseExpression(node) + of "shift_expression": + result = exprParser.processShiftExpression(node) + of "logical_expression": + result = exprParser.processLogicalExpression(node) + of "identifier": + var ident = node.val + if ident != "_": + ident = exprParser.state.getIdentifier(ident, nskConst) + result = exprParser.state.getIdent(ident) + else: + result = newNode(nkNilLit) + + decho "# NODERES: ", result + +proc codeToNode*(state: NimState, code: string): PNode = + let exprParser = newExprParser(state, code) + withCodeAst(exprParser): + result = exprParser.processTSNode(root) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 4ffb5d7..b96ae70 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -221,7 +221,7 @@ proc len*(node: TSNode): int = result = node.tsNodeNamedChildCount().int proc `[]`*(node: TSNode, i: SomeInteger): TSNode = - if i < node.len: + if i < type(i)(node.len()): result = node.tsNodeNamedChild(i.uint32) proc getName*(node: TSNode): string {.inline.} = From c1520b4fd103da78c15c807092f7bcbb46d16012 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 06:33:02 -0600 Subject: [PATCH 064/255] Add better error handling for unimplemented types --- nimterop/exprparser.nim | 75 ++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a4be5d0..9570788 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -13,13 +13,15 @@ type state*: NimState code*: string + ExprParseError* = object of CatchableError + proc newExprParser*(state: NimState, code: string): ExprParser = ExprParser(state: state, code: code) -template decho(msg: varargs[string, `$`]) = +template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: let nimState {.inject.} = exprParser.state - necho "# " & join(msg, "") + necho "# " & join(msg, "").replace("\n", "\n# ") template val*(node: TSNode): string = exprParser.code.getNodeVal(node) @@ -54,16 +56,19 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result = newNode(nkNilLit) if number.contains("."): let floatSuffix = number[result.len-1] - case floatSuffix - of 'l', 'L': - # TODO: handle long double (128 bits) - # result = newNode(nkFloat128Lit) - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - of 'f', 'F': - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - else: - discard - return + try: + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1])) + return + except ValueError: + raise newException(ExprParseError, &"Could not parse float value \"{number}\".") case suffix of "u", "U": @@ -111,6 +116,8 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = exprParser.state.getIdent("-"), result ) + else: + raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkCharLit) @@ -135,7 +142,7 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = of ">>": result.add exprParser.state.getIdent("shr") else: - discard + raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") result.add exprParser.processTSNode(left) result.add exprParser.processTSNode(right) @@ -151,15 +158,15 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - decho "LOG SYM: ", binarySym + techo "LOG SYM: ", binarySym case binarySym of "!": nimSym = "not" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"") - decho "LOG CHILD: ", child.val, ", nim: ", nimSym + techo "LOG CHILD: ", child.val, ", nim: ", nimSym result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), exprParser.processTSNode(child) @@ -173,7 +180,7 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() - decho "# BIN SYM: ", binarySym + techo "BIN SYM: ", binarySym case binarySym of "|", "||": @@ -183,7 +190,7 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = of "^": nimSym = "xor" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) result.add exprParser.processTSNode(left) @@ -195,23 +202,26 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - decho "# BIN SYM: ", binarySym + techo "BIN SYM: ", binarySym case binarySym of "~": nimSym = "not" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"") result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), exprParser.processTSNode(child) ) + else: + raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkNilLit) - decho "# NODE: ", node.getName(), ", VAL: ", node.val - case node.getName() + let nodeName = node.getName() + techo "NODE: ", nodeName, ", VAL: ", node.val + case nodeName of "number_literal": result = exprParser.processNumberLiteral(node) of "string_literal": @@ -220,7 +230,11 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = result = exprParser.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # This may be wrong. What can be in an expression? - result = exprParser.processTSNode(node[0]) + if node.len > 0: + result = exprParser.processTSNode(node[0]) + else: + raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") + of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node) of "bitwise_expression": @@ -235,11 +249,18 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = ident = exprParser.state.getIdentifier(ident, nskConst) result = exprParser.state.getIdent(ident) else: - result = newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - decho "# NODERES: ", result + techo "NODERES: ", result proc codeToNode*(state: NimState, code: string): PNode = let exprParser = newExprParser(state, code) - withCodeAst(exprParser): - result = exprParser.processTSNode(root) \ No newline at end of file + try: + withCodeAst(exprParser): + result = exprParser.processTSNode(root) + except ExprParseError as e: + techo e.msg + result = newNode(nkNilLit) + except Exception as e: + techo e.msg + result = newNode(nkNilLit) \ No newline at end of file From 305d90583e5dc7a8dde012c982047df9dee21f8e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 06:53:41 -0600 Subject: [PATCH 065/255] Add proper binary casting --- nimterop/exprparser.nim | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 9570788..1f72a9a 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -144,8 +144,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - result.add exprParser.processTSNode(left) - result.add exprParser.processTSNode(right) + let + leftNode = exprParser.processTSNode(left) + rightNode = exprParser.processTSNode(right) + + result.add leftNode + result.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ), + rightNode + ) proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkPar) @@ -193,8 +203,18 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) - result.add exprParser.processTSNode(left) - result.add exprParser.processTSNode(right) + let + leftNode = exprParser.processTSNode(left) + rightNode = exprParser.processTSNode(right) + + result.add leftNode + result.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ), + rightNode + ) elif node.len() == 1: result = newNode(nkPar) From 055d6bee73eb988ae498a57d29dd3d80a8928721 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 07:37:17 -0600 Subject: [PATCH 066/255] Add math_expression and fix ast2 tests Update to include more expressions --- nimterop/ast2.nim | 24 +---- nimterop/exprparser.nim | 200 ++++++++++++++++++++++++++++++---------- nimterop/getters.nim | 22 +++-- nimterop/globals.nim | 2 +- tests/include/tast2.h | 18 ++++ tests/tast2.nim | 22 ++++- 6 files changed, 211 insertions(+), 77 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 04bb8eb..85b555f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,10 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser] +import "."/[globals, getters, exprparser, utils] proc getPtrType*(str: string): string = result = case str: @@ -17,21 +17,6 @@ proc getPtrType*(str: string): string = else: str -proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = - # Raise exception in parseString() instead of exiting for errors - if msg < warnMin: - raise newException(Exception, msgKindToString(msg)) - -proc parseString(gState: State, str: string): PNode = - # Parse a string into Nim AST - use custom error handler that raises - # an exception rather than exiting on failure - try: - result = parseString( - str, gState.identCache, gState.config, errorHandler = handleError - ) - except: - decho getCurrentExceptionMsg() - proc getLit*(nimState: NimState, str: string, expression = false): PNode = result = nimState.codeToNode(str) @@ -121,7 +106,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind != nkNilLit: + elif valident.kind != nkNone: if nimState.addNewIdentifer(name): # const X* = Y # @@ -179,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - let + var colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1501,6 +1486,7 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") + var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f72a9a..505453c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters] +import "."/[globals, getters, utils] type ExprParser* = ref object @@ -21,15 +21,16 @@ proc newExprParser*(state: NimState, code: string): ExprParser = template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: let nimState {.inject.} = exprParser.state - necho "# " & join(msg, "").replace("\n", "\n# ") + necho join(msg, "").getCommented -template val*(node: TSNode): string = +template val(node: TSNode): string = exprParser.code.getNodeVal(node) -proc mode*(exprParser: ExprParser): string = +proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode template withCodeAst(exprParser: ExprParser, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() defer: parser.tsParserDelete() @@ -51,9 +52,9 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() - proc getNumNode(number, suffix: string): PNode {.inline.} = - result = newNode(nkNilLit) + ## Convert a C number to a Nim number PNode + result = newNode(nkNone) if number.contains("."): let floatSuffix = number[result.len-1] try: @@ -84,6 +85,8 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result = newNode(nkIntLit) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. if number.contains(re"0[xX]"): result.intVal = parseHexInt(number) result.flags = {nfBase16} @@ -97,11 +100,11 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) + result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -111,7 +114,7 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = getNumNode(number, suffix) - if result.kind != nkNilLit and prefix == "-": + if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( exprParser.state.getIdent("-"), result @@ -127,9 +130,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkInfix) let left = node[0] @@ -144,25 +147,29 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode result.add nkCast.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i])) + result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -179,14 +186,96 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = techo "LOG CHILD: ", child.val, ", nim: ", nimSym result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) -proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + if node.len > 1: + # Node has left and right children ie: (2 + 7) + var + res = newNode(nkInfix) + let + left = node[0] + right = node[1] + + let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + techo "MATH SYM: ", mathSym + + res.add exprParser.state.getIdent(mathSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) + + res.add leftNode + # res.add rightNode + res.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + rightNode + ) + + # Make sure the statement is of the same type as the left + # hand argument, since some expressions return a differing + # type than the input types (2/3 == float) + result = nkCall.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + res + ) + + elif node.len() == 1: + # Node has only one child, ie -(20 + 7) + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "MATH SYM: ", unarySym + + case unarySym + of "+": + nimSym = "+" + of "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gental cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + return + else: + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + else: + raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = if node.len() > 1: result = newNode(nkInfix) - let left = node[0] - let right = node[1] + + let + left = node[0] + right = node[1] + var nimSym = "" var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() @@ -203,15 +292,19 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) @@ -221,24 +314,26 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - techo "BIN SYM: ", binarySym + var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "BIN SYM: ", unarySym - case binarySym + case unarySym of "~": nimSym = "not" else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + ## Handle all of the types of expressions here. This proc gets called recursively + ## in the processX procs and will drill down to sub nodes. + result = newNode(nkNone) let nodeName = node.getName() techo "NODE: ", nodeName, ", VAL: ", node.val case nodeName @@ -251,36 +346,47 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = of "expression_statement", "ERROR", "translation_unit": # This may be wrong. What can be in an expression? if node.len > 0: - result = exprParser.processTSNode(node[0]) + result = exprParser.processTSNode(node[0], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") - of "parenthesized_expression": - result = exprParser.processParenthesizedExpr(node) + result = exprParser.processParenthesizedExpr(node, typeofNode) of "bitwise_expression": - result = exprParser.processBitwiseExpression(node) + result = exprParser.processBitwiseExpression(node, typeofNode) + of "math_expression": + result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": - result = exprParser.processShiftExpression(node) + result = exprParser.processShiftExpression(node, typeofNode) of "logical_expression": - result = exprParser.processLogicalExpression(node) + result = exprParser.processLogicalExpression(node, typeofNode) + # Why are these node types named true/false? + of "true", "false": + result = exprParser.state.parseString(node.val) of "identifier": var ident = node.val if ident != "_": + # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, nskConst) - result = exprParser.state.getIdent(ident) + techo ident + if ident != "": + result = exprParser.state.getIdent(ident) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODERES: ", result + techo "NODE RES: ", result proc codeToNode*(state: NimState, code: string): PNode = - let exprParser = newExprParser(state, code) + ## Convert the C string to a nim PNode tree + result = newNode(nkNone) try: + let exprParser = newExprParser(state, code) withCodeAst(exprParser): result = exprParser.processTSNode(root) except ExprParseError as e: - techo e.msg - result = newNode(nkNilLit) + echo e.msg.getCommented + result = newNode(nkNone) except Exception as e: - techo e.msg - result = newNode(nkNilLit) \ No newline at end of file + echo e.msg.getCommented + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b96ae70..f43f199 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -399,7 +399,7 @@ proc printLisp*(code: var string, root: TSNode): string = else: break - if node.tsNodeNamedChildCount() != 0: + if node.len() != 0: result &= "\n" nextnode = node.tsNodeNamedChild(0) depth += 1 @@ -459,15 +459,19 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(gState: State, node: TSNode) = - if gState.debug: - gecho ("Input => " & gState.getNodeVal(node)).getCommented() & "\n" & - gState.printLisp(node).getCommented() +proc printDebug*(nimState: NimState, node: TSNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug: + necho ("Input => " & nimState.getNodeVal(node)).getCommented() + necho nimState.gState.printLisp(node).getCommented() -proc printDebug*(gState: State, pnode: PNode) = - if gState.debug: - gecho ("Output => " & $pnode).getCommented() & "\n" & - gState.printTree(pnode) +proc printDebug*(nimState: NimState, pnode: PNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug and pnode.kind != nkNone: + necho ("Output => " & $pnode).getCommented() + necho nimState.printTree(pnode) # Compiler shortcuts diff --git a/nimterop/globals.nim b/nimterop/globals.nim index f159124..b964da9 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -113,7 +113,7 @@ when not declared(CIMPORT): export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl # Redirect output to file when required - template gecho*(args: string) {.dirty.} = + template gecho*(args: string) = if gState.outputHandle.isNil: echo args else: diff --git a/tests/include/tast2.h b/tests/include/tast2.h index bdf8823..7e15ac0 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -8,6 +8,24 @@ extern "C" { #define D "hello" #define E 'c' +#define UEXPR (1234u << 1) +#define ULEXPR (1234ul << 2) +#define ULLEXPR (1234ull << 3) +#define LEXPR (1234l << 4) +#define LLEXPR (1234ll << 5) + +#define SHL1 (1u << 1) +#define SHL2 (1u << 2) +#define SHL3 (1u << 3) +#define COERCE 645635634896ull + -35436 +#define COERCE2 645635634896 + -35436 +#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define BOOL true +#define MATHEXPR (1 + 2/3*20 - 100) +#define ANDEXPR (100 & 11000) + +#define ALLSHL (SHL1 | SHL2 | SHL3) + struct A0; struct A1 {}; typedef struct A2; diff --git a/tests/tast2.nim b/tests/tast2.nim index e13c4ac..d65e34a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -105,6 +105,26 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert UEXPR == (1234.uint shl 1) +assert ULEXPR == (1234.uint32 shl 2) +assert ULLEXPR == (1234.uint64 shl 3) +assert LEXPR == (1234.int32 shl 4) +assert LLEXPR == (1234.int64 shl 5) + +assert COERCE == 645635599460'u64 +assert COERCE2 == 645635599460'i64 + +assert BINEXPR == 5 +assert BOOL == true +assert MATHEXPR == -99 +assert ANDEXPR == 96 + +assert SHL1 == (1.uint shl 1) +assert SHL2 == (1.uint shl 2) +assert SHL3 == (1.uint shl 3) + +assert ALLSHL == (SHL1 or SHL2 or SHL3) + assert A0 is object testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) @@ -271,7 +291,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[123 + 132, ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 62c68d69ee0409558a846aae3a85cf1a29f9442b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 18 Apr 2020 13:13:33 -0600 Subject: [PATCH 067/255] Add sizeof expression parser, fix nkNone issue, reduce size of typeof expressions Cleanup, fix bugs --- nimterop/ast2.nim | 2 +- nimterop/exprparser.nim | 149 +++++++++++++++++++++++++--------------- nimterop/getters.nim | 4 +- 3 files changed, 97 insertions(+), 58 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 85b555f..8c5da8c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -963,7 +963,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo let # Size of array could be a Nim expression size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) - if size.kind != nkNilLit: + if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] elif cnode.len == 1: diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 505453c..d27f23b 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -29,6 +29,24 @@ template val(node: TSNode): string = proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode +proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + result = newNode(nkNone) + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = exprParser.state.getIdentifier(ident, kind, parent) + if ident != "": + result = exprParser.state.getIdent(ident) + +proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + exprParser.getIdent(node.val, kind, parent) + template withCodeAst(exprParser: ExprParser, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() @@ -56,7 +74,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[result.len-1] + let floatSuffix = number[number.len-1] try: case floatSuffix of 'l', 'L': @@ -66,7 +84,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = of 'f', 'F': result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: - result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1])) + result = newFloatNode(nkFloatLit, parseFloat(number)) return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") @@ -100,11 +118,12 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = + ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -130,9 +149,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -149,27 +168,35 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: P let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCast.newTree( + exprParser.processTSNode(node[0], typeofNode), + exprParser.processTSNode(node[1], typeofNode) + ) + +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -189,7 +216,7 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: exprParser.processTSNode(child, typeofNode) ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) var @@ -204,19 +231,20 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo res.add exprParser.state.getIdent(mathSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - # res.add rightNode res.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -224,10 +252,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # hand argument, since some expressions return a differing # type than the input types (2/3 == float) result = nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, res ) @@ -248,6 +273,8 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # so we have to make a gental cast here to coerce it to one. # Might be bad because we are overwriting the type # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") result.add nkPrefix.newTree( exprParser.state.getIdent(unarySym), nkPar.newTree( @@ -268,7 +295,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len() > 1: result = newNode(nkInfix) @@ -288,24 +315,25 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: nimSym = "and" of "^": nimSym = "xor" + of "==", "!=": + nimSym = binarySym else: raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -317,6 +345,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() techo "BIN SYM: ", unarySym + # TODO: Support more symbols here case unarySym of "~": nimSym = "not" @@ -330,7 +359,13 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCall.newTree( + exprParser.state.getIdent("sizeof"), + exprParser.processTSNode(node[0], typeofNode) + ) + +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) @@ -351,42 +386,48 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node, typeofNode) - of "bitwise_expression": + of "sizeof_expression": + result = exprParser.processSizeofExpression(node, typeofNode) + of "bitwise_expression", "equality_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": result = exprParser.processShiftExpression(node, typeofNode) + of "cast_expression": + result = exprParser.processCastExpression(node, typeofNode) of "logical_expression": result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "identifier": - var ident = node.val - if ident != "_": - # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, nskConst) - techo ident - if ident != "": - result = exprParser.state.getIdent(ident) + of "type_descriptor": + let ty = getType(node.val) + result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") + result = exprParser.state.getIdent(ty) + of "identifier": + result = exprParser.getIdent(node, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RES: ", result + techo "NODE RESULT: ", result proc codeToNode*(state: NimState, code: string): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) + # This is used for keeping track of the type of the first + # symbol + var tnode: PNode = nil + let exprParser = newExprParser(state, code) try: - let exprParser = newExprParser(state, code) withCodeAst(exprParser): - result = exprParser.processTSNode(root) + result = exprParser.processTSNode(root, tnode) except ExprParseError as e: - echo e.msg.getCommented + techo e.msg result = newNode(nkNone) except Exception as e: - echo e.msg.getCommented + techo "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f43f199..216d2e6 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -460,18 +460,16 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(nimState: NimState, node: TSNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug: necho ("Input => " & nimState.getNodeVal(node)).getCommented() necho nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug and pnode.kind != nkNone: necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode) + necho nimState.printTree(pnode).getCommented() # Compiler shortcuts From 173e6d625c3ea96a95580303fedd011684635cc4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:18:04 -0600 Subject: [PATCH 068/255] Add string and char support Update some comments Rename exprparser main proc Don't export parse procs Add missing utils module Try to fix array type test Try fix cast test error Disable cast test for now Revert back comment test. Have to figure out how to test without vm --- nimterop/ast2.nim | 6 +-- nimterop/exprparser.nim | 101 +++++++++++++++++++++++++++++++++------- nimterop/utils.nim | 18 +++++++ tests/include/tast2.h | 6 +++ tests/tast2.nim | 12 +++-- 5 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 nimterop/utils.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8c5da8c..08ea2d6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -18,7 +18,7 @@ proc getPtrType*(str: string): string = str proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.codeToNode(str) + result = nimState.parseCExpression(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -164,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - var + let colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index d27f23b..183bb18 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros +import strformat, strutils, macros, sets import regex @@ -12,11 +12,12 @@ type ExprParser* = ref object state*: NimState code*: string + name*: string ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string): ExprParser = - ExprParser(state: state, code: code) +proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = + ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: @@ -38,6 +39,8 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) + if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: + ident = ident & "." & exprParser.name if ident != "": result = exprParser.state.getIdent(ident) @@ -70,6 +73,58 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() +proc parseChar(charStr: string): uint8 {.inline.} = + ## Parses a character literal out of a string. This is needed + ## because treesitter gives unescaped characters when parsing + ## strings. + if charStr.len == 1: + return charStr[0].uint8 + + # Handle octal, hex, unicode? + if charStr.startsWith("\\x"): + result = parseHexInt(charStr.replace("\\x", "0x")).uint8 + elif charStr.len == 4: # Octal + result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8 + + if result == 0: + case charStr + of "\\0": + result = ord('\0') + of "\\a": + result = 0x07 + of "\\b": + result = 0x08 + of "\\e": + result = 0x1B + of "\\f": + result = 0x0C + of "\\n": + result = '\n'.uint8 + of "\\r": + result = 0x0D + of "\\t": + result = 0x09 + of "\\v": + result = 0x0B + of "\\\\": + result = 0x5C + of "\\'": + result = '\''.uint8 + of "\\\"": + result = '\"'.uint8 + of "\\?": + result = 0x3F + else: + discard + + if result > uint8.high: + result = uint8.high + +proc getCharLit(charStr: string): PNode {.inline.} = + ## Convert a character string into a proper Nim char lit node + result = newNode(nkCharLit) + result.intVal = parseChar(charStr).int64 + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) @@ -117,7 +172,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result.intVal = parseInt(number) -proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -141,17 +196,29 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkCharLit) - result.intVal = node.val[1].int64 +proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = + let val = node.val + result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = - let nodeVal = node.val - result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) +proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = + let + nodeVal = node.val + strVal = nodeVal[1 ..< nodeVal.len - 1] -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode + const + str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + reg = re(str) -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + # Convert the c string escape sequences/etc to Nim chars + var nimStr = newStringOfCap(nodeVal.len) + for m in strVal.findAll(reg): + nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) + + result = newStrNode(nkStrLit, nimStr) + +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode + +proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -185,18 +252,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -415,13 +482,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): techo "NODE RESULT: ", result -proc codeToNode*(state: NimState, code: string): PNode = +proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol var tnode: PNode = nil - let exprParser = newExprParser(state, code) + let exprParser = newExprParser(state, code, name) try: withCodeAst(exprParser): result = exprParser.processTSNode(root, tnode) diff --git a/nimterop/utils.nim b/nimterop/utils.nim new file mode 100644 index 0000000..025256a --- /dev/null +++ b/nimterop/utils.nim @@ -0,0 +1,18 @@ +import compiler/[ast, lineinfos, msgs, options, parser, renderer] + +import "."/[globals, getters] + +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting for errors + if msg < warnMin: + raise newException(Exception, msgKindToString(msg)) + +proc parseString*(nimState: NimState, str: string): PNode = + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 7e15ac0..3c6148a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,6 +23,12 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) +#define CASTEXPR (int) 34 + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) diff --git a/tests/tast2.nim b/tests/tast2.nim index d65e34a..d00552c 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -93,11 +93,11 @@ macro testFields(t: typed, fields: static[string] = "") = for i in 0 ..< rl.len: let name = ($rl[i][0]).strip(chars = {'*'}) - typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "") + typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "").replace("typeof", "type") n = names.find(name) assert n != -1, $t & "." & name & " invalid" - assert types[n] == typ, - "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ + assert types[n].replace("typeof", "type") == typ, + "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ assert A == 2 assert B == 1.0 @@ -118,6 +118,12 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 +assert CASTEXPR == 34.chr + +assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" +assert NULLCHAR == '\0' +assert OCTCHAR == '\n' +assert HEXCHAR.int == 0xFE assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) From aa1086fae37999180b485ced323d35acec7868a1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 09:08:08 -0600 Subject: [PATCH 069/255] Fix cast type only run tast2 on supported compilers Change cast to gentle type cast Re enable tests for Nim < 1.0.0 --- nimterop/exprparser.nim | 6 +++--- tests/include/tast2.h | 6 +++--- tests/tast2.nim | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 183bb18..ed287a3 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -247,7 +247,7 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( typeofNode, rightNode ) @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCast.newTree( + result = nkCall.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) @@ -310,7 +310,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - res.add nkCast.newTree( + res.add nkCall.newTree( typeofNode, rightNode ) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 3c6148a..53b4d64 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -17,13 +17,13 @@ extern "C" { #define SHL1 (1u << 1) #define SHL2 (1u << 2) #define SHL3 (1u << 3) -#define COERCE 645635634896ull + -35436 -#define COERCE2 645635634896 + -35436 +#define COERCE 645635634896ull + 35436 +#define COERCE2 645635634896 + 35436ul #define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -#define CASTEXPR (int) 34 +#define CASTEXPR (char) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index d00552c..d43ce9f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -111,8 +111,8 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert COERCE == 645635599460'u64 -assert COERCE2 == 645635599460'i64 +assert COERCE == 645635670332'u64 +assert COERCE2 == 645635670332'i64 assert BINEXPR == 5 assert BOOL == true @@ -297,7 +297,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 208098b3eb959e477a3625a8a48305ddaa1cec4f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 11:33:53 -0600 Subject: [PATCH 070/255] Make bitwise_expression forwards compatible Skip syms for tmath Skip more syms Add casting back for cast_expression. Disable cast test on Nim < 1.0.0 Skip more syms for cast expressions on Nim < 1.0.0 --- nimterop/exprparser.nim | 8 +++++--- tests/tast2.nim | 11 +++++++++-- tests/tmath.nim | 4 ++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index ed287a3..207d7fd 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCall.newTree( + result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) @@ -455,7 +455,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": result = exprParser.processSizeofExpression(node, typeofNode) - of "bitwise_expression", "equality_expression": + # binary_expression from the new treesitter upgrade should work here + # once we upgrade + of "bitwise_expression", "equality_expression", "binary_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) @@ -497,4 +499,4 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + result = newNode(nkNone) diff --git a/tests/tast2.nim b/tests/tast2.nim index d43ce9f..0b3762a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -3,11 +3,16 @@ import macros, os, sets, strutils import nimterop/[cimport] static: + # Skip casting on lower nim compilers because + # the VM does not support it + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["CASTEXPR"] cDebug() const path = currentSourcePath.parentDir() / "include" / "tast2.h" + when defined(HEADER): cDefine("HEADER") const @@ -118,7 +123,9 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -assert CASTEXPR == 34.chr + +when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0): + assert CASTEXPR == 34.chr assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' @@ -453,4 +460,4 @@ checkPragmas(nested, pHeaderImpBy) when defined(HEADER): assert sitest1(5) == 10 - assert sitest1(10) == 20 \ No newline at end of file + assert sitest1(10) == 20 diff --git a/tests/tmath.nim b/tests/tmath.nim index 5d84700..bbf7abd 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,6 +13,10 @@ when defined(windows): complex = object static: + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] + else: + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] cDebug() cDisableCaching() cAddStdDir() From 1576127a6e1b5f429d3fb91cf924fc41e43346cf Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:09:08 -0600 Subject: [PATCH 071/255] Use tsNodeChild --- nimterop/exprparser.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 207d7fd..610c0cb 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -223,7 +223,8 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let left = node[0] right = node[1] - var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + + let shiftSym = node.tsNodeChild(1).val.strip() case shiftSym of "<<": @@ -268,7 +269,7 @@ proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(0).val.strip() techo "LOG SYM: ", binarySym case binarySym @@ -292,7 +293,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var left = node[0] right = node[1] - let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let mathSym = node.tsNodeChild(1).val.strip() techo "MATH SYM: ", mathSym res.add exprParser.state.getIdent(mathSym) @@ -329,7 +330,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let child = node[0] var nimSym = "" - let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "MATH SYM: ", unarySym case unarySym @@ -372,7 +373,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var nimSym = "" - var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(1).val.strip() techo "BIN SYM: ", binarySym case binarySym @@ -409,7 +410,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym # TODO: Support more symbols here @@ -437,7 +438,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() + techo "NODE: ", nodeName, ", VAL: ", node.val + case nodeName of "number_literal": result = exprParser.processNumberLiteral(node) From a84adfe1d8db39e91f4e0e0af4cd81f02cee2fd6 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:55:05 -0600 Subject: [PATCH 072/255] Add comments describing capabilities --- nimterop/exprparser.nim | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 610c0cb..1fc72ab 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -8,6 +8,27 @@ import "."/treesitter/[api, c, cpp] import "."/[globals, getters, utils] +# This version of exprparser should be able to handle: +# +# All integers + integer like expressions (hex, octal, suffixes) +# All floating point expressions (except for C++'s hex floating point stuff) +# Strings and character literals, including C's escape characters (not sure if this is the same as C++'s escape characters or not) +# Math operators (+, -, /, *) +# Some Unary operators (-, !, ~). ++, --, and & are yet to be implemented +# Any identifiers +# C type descriptors (int, char, etc) +# Boolean values (true, false) +# Shift expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Math expressions (containing anything in this list) +# Sizeof expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Parentheses expressions (containing anything in this list) +# Expressions containing other expressions +# +# In addition to the above, it should also handle most type coercions, except +# for where Nim can't (such as uint + -int) + type ExprParser* = ref object state*: NimState @@ -413,7 +434,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym - # TODO: Support more symbols here + # TODO: Support more symbols here. ++, --, & case unarySym of "~": nimSym = "not" From d83884a3917cf96b7df209d47c7301919cce3967 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 18:04:12 -0600 Subject: [PATCH 073/255] Add missing sized type specifier for 'long int', etc --- nimterop/exprparser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1fc72ab..44e59d6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -494,7 +494,7 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "type_descriptor": + of "type_descriptor", "sized_type_specifier": let ty = getType(node.val) result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: From 4794c076ffc7f46858dfebfdae47d8e3c47d1cf7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 19:01:53 -0600 Subject: [PATCH 074/255] Fix pcre tests --- tests/tpcre.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tpcre.nim b/tests/tpcre.nim index c8e8059..3bbd1ba 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,6 +7,7 @@ const pcreH = baseDir/"pcre.h.in" static: + cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From b1a445c34dfff04d5c799b7c4ee0f7e806900e91 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 01:16:19 -0600 Subject: [PATCH 075/255] Update based on comments from review. Need to add more docs and reorg to use gstate --- nimterop/ast2.nim | 30 +- nimterop/{utils.nim => comphelp.nim} | 0 nimterop/exprparser.nim | 394 ++++++++++++--------------- nimterop/getters.nim | 20 +- nimterop/toast.nim | 20 +- nimterop/tshelp.nim | 30 ++ tests/tast2.nim | 1 - 7 files changed, 249 insertions(+), 246 deletions(-) rename nimterop/{utils.nim => comphelp.nim} (100%) create mode 100644 nimterop/tshelp.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 08ea2d6..1b7fb3d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,12 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times +import options as opts + import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser, utils] +import "."/[globals, getters, exprparser, comphelp] proc getPtrType*(str: string): string = result = case str: @@ -17,9 +19,6 @@ proc getPtrType*(str: string): string = else: str -proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.parseCExpression(str) - proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined # and use that if present @@ -101,7 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gState.getNodeVal(node[1]) valident = - gState.getLit(val) + gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank @@ -962,7 +961,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo # type name[X] => array[X, type] let # Size of array could be a Nim expression - size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) + size = gState.parseCExpression(gState.getNodeVal(cnode[1])) if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] @@ -1367,6 +1366,7 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] + fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1385,20 +1385,25 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - # Explicit value - fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name - - # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + fvalSections.add((fname, "", some(en[1]))) + else: + fvalSections.add((fname, fval, none(TSNode))) fnames.incl fname - prev = fname # Add fields to list of consts after processing enum so that we don't cast # enum field to itself gState.constIdentifiers.incl fnames + # parseCExpression requires all const identifiers to be present for the enum + for (fname, fval, cexprNode) in fvalSections: + var fval = fval + if cexprNode.isSome: + fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) @@ -1486,7 +1491,6 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") - var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/utils.nim b/nimterop/comphelp.nim similarity index 100% rename from nimterop/utils.nim rename to nimterop/comphelp.nim diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 44e59d6..2ce4989 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, utils] +import "."/[globals, getters, comphelp, tshelp] # This version of exprparser should be able to handle: # @@ -41,9 +41,9 @@ proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = - if exprParser.state.gState.debug: + block: let nimState {.inject.} = exprParser.state - necho join(msg, "").getCommented + decho join(msg, "") template val(node: TSNode): string = exprParser.code.getNodeVal(node) @@ -60,9 +60,11 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) - if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: - ident = ident & "." & exprParser.name - if ident != "": + if kind == nskType: + result = exprParser.state.getIdent(ident) + elif ident.nBl and ident in exprParser.state.constIdentifiers: + if exprParser.name.nBl: + ident = ident & "." & exprParser.name result = exprParser.state.getIdent(ident) proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = @@ -71,29 +73,6 @@ proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = "" ## Returns PNode(nkNone) if the identifier is blank exprParser.getIdent(node.val, kind, parent) -template withCodeAst(exprParser: ExprParser, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code - var parser = tsParserNew() - defer: - parser.tsParserDelete() - - doAssert exprParser.code.nBl, "Empty code" - if exprParser.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif exprParser.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {exprParser.mode}" - - var - tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32) - root {.inject.} = tree.tsTreeRootNode() - - body - - defer: - tree.tsTreeDelete() - proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed ## because treesitter gives unescaped characters when parsing @@ -161,37 +140,36 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: result = newFloatNode(nkFloatLit, parseFloat(number)) - return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") - - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) else: - result = newNode(nkIntLit) + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc @@ -285,168 +263,112 @@ proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let binarySym = node.tsNodeChild(0).val.strip() - techo "LOG SYM: ", binarySym - - case binarySym - of "!": - nimSym = "not" +proc getNimUnarySym(csymbol: string): string = + ## Get the Nim equivalent of a unary C symbol + ## + ## TODO: Add ++, --, + case csymbol + of "+", "-": + result = csymbol + of "~", "!": + result = "not" else: - raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{csymbol}\"") - techo "LOG CHILD: ", child.val, ", nim: ", nimSym - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) +proc getNimBinarySym(csymbol: string): string = + case csymbol + of "|", "||": + result = "or" + of "&", "&&": + result = "and" + of "^": + result = "xor" + of "==", "!=", + "+", "-", "/", "*", + ">", "<", ">=", "<=": + result = csymbol + of "%": + result = "mod" + else: + raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") + +proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + # Node has left and right children ie: (2 + 7) + result = newNode(nkInfix) + + let + left = node[0] + right = node[1] + binarySym = node.tsNodeChild(1).val.strip() + nimSym = getNimBinarySym(binarySym) + + result.add exprParser.state.getIdent(nimSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) + + let rightNode = exprParser.processTSNode(right, typeofNode) + + result.add leftNode + result.add nkCall.newTree( + typeofNode, + rightNode ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = newNode(nkPar) + + let + child = node[0] + unarySym = node.tsNodeChild(0).val.strip() + nimSym = getNimUnarySym(unarySym) + + if nimSym == "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gentle cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + else: + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + +proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) - var - res = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let mathSym = node.tsNodeChild(1).val.strip() - techo "MATH SYM: ", mathSym - - res.add exprParser.state.getIdent(mathSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - res.add leftNode - res.add nkCall.newTree( - typeofNode, - rightNode - ) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) + let binExpr = processBinaryExpression(exprParser, node, typeofNode) + # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, - res + binexpr ) elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "MATH SYM: ", unarySym - - case unarySym - of "+": - nimSym = "+" - of "-": - # Special case. The minus symbol must be in front of an integer, - # so we have to make a gental cast here to coerce it to one. - # Might be bad because we are overwriting the type - # There's probably a better way of doing this - if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") - result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), - nkPar.newTree( - nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) - ) - ) - ) - return - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) + result = processUnaryExpression(exprParser, node, typeofNode) else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") - -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - if node.len() > 1: - result = newNode(nkInfix) - - let - left = node[0] - right = node[1] - - var nimSym = "" - - let binarySym = node.tsNodeChild(1).val.strip() - techo "BIN SYM: ", binarySym - - case binarySym - of "|", "||": - nimSym = "or" - of "&", "&&": - nimSym = "and" - of "^": - nimSym = "xor" - of "==", "!=": - nimSym = binarySym - else: - raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") - - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - - elif node.len() == 1: - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "BIN SYM: ", unarySym - - # TODO: Support more symbols here. ++, --, & - case unarySym - of "~": - nimSym = "not" - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) - else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( @@ -464,45 +386,85 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): case nodeName of "number_literal": + # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f + # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 result = exprParser.processNumberLiteral(node) of "string_literal": + # Input -> "foo\0\x42" + # Output -> "foo\0" result = exprParser.processStringLiteral(node) of "char_literal": + # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> result = exprParser.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": - # This may be wrong. What can be in an expression? - if node.len > 0: + # Note that we're parsing partial expressions, so the TSNode might contain + # an ERROR node. If that's the case, they usually contain children with + # partial results, which will contain parsed expressions + # + # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) + # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ + if node.len == 1: result = exprParser.processTSNode(node[0], typeofNode) + elif node.len > 1: + result = newNode(nkStmtListExpr) + for i in 0 ..< node.len: + result.add exprParser.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": + # Input -> (IDENT - OTHERIDENT) + # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": + # Input -> sizeof(char) + # Output -> sizeof(cchar) result = exprParser.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade - of "bitwise_expression", "equality_expression", "binary_expression": - result = exprParser.processBitwiseExpression(node, typeofNode) - of "math_expression": - result = exprParser.processMathExpression(node, typeofNode) + of "math_expression", "logical_expression", "relational_expression", + "bitwise_expression", "equality_expression", "binary_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + # Output -> + # typeof(a)(a == typeof(a)(b)) + # typeof(a)(a != typeof(a)(b)) + # (not a) + # (not a) + # typeof(a)(a < typeof(a)(b)) + # typeof(a)(a > typeof(a)(b)) + # typeof(a)(a <= typeof(a)(b)) + # typeof(a)(a >= typeof(a)(b)) + result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": + # Input -> a >> b, a << b + # Output -> a shr typeof(a)(b), a shl typeof(a)(b) result = exprParser.processShiftExpression(node, typeofNode) of "cast_expression": + # Input -> (int) a + # Output -> cast[cint](a) result = exprParser.processCastExpression(node, typeofNode) - of "logical_expression": - result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": + # Input -> true, false + # Output -> true, false result = exprParser.state.parseString(node.val) of "type_descriptor", "sized_type_specifier": + # Input -> int, unsigned int, long int, etc + # Output -> cint, cuint, clong, etc let ty = getType(node.val) - result = exprParser.getIdent(ty, nskType, parent=node.getName()) - if result.kind == nkNone: - result = exprParser.state.getIdent(ty) + if ty.len > 0: + # If ty is not empty, one of C's builtin types has been found + result = exprParser.getIdent(ty, nskType, parent=node.getName()) + else: + result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": + # Input -> IDENT + # Output -> IDENT (if found in sym table, else error) result = exprParser.getIdent(node, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") + raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") @@ -512,15 +474,15 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first - # symbol + # symbol used for type casting var tnode: PNode = nil let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser): + withCodeAst(exprParser.code, exprParser.mode): result = exprParser.processTSNode(root, tnode) except ExprParseError as e: techo e.msg result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 216d2e6..b7744ce 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -401,7 +401,7 @@ proc printLisp*(code: var string, root: TSNode): string = if node.len() != 0: result &= "\n" - nextnode = node.tsNodeNamedChild(0) + nextnode = node[0] depth += 1 else: result &= ")\n" @@ -432,7 +432,7 @@ proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") proc printTree*(gState: State, pnode: PNode, offset = ""): string = - if gState.debug and pnode.kind != nkNone: + if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: @@ -459,17 +459,17 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(nimState: NimState, node: TSNode) = +proc printDebug*(gState: State, node: TSNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug: - necho ("Input => " & nimState.getNodeVal(node)).getCommented() - necho nimState.gState.printLisp(node).getCommented() + if gState.debug: + gecho ("Input => " & gState.getNodeVal(node)).getCommented() + gecho gState.printLisp(node).getCommented() -proc printDebug*(nimState: NimState, pnode: PNode) = +proc printDebug*(gState: State, pnode: PNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug and pnode.kind != nkNone: - necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode).getCommented() + if gState.debug and pnode.kind != nkNone: + gecho ("Output => " & $pnode).getCommented() + gecho gState.printTree(pnode).getCommented() # Compiler shortcuts diff --git a/nimterop/toast.nim b/nimterop/toast.nim index ab44970..afb511d 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,16 +2,11 @@ import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, globals, getters, grammar, build] +import "."/[ast, ast2, globals, getters, grammar, build, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" - var parser = tsParserNew() - - defer: - parser.tsParserDelete() - if gState.mode.Bl: gState.mode = getCompilerMode(path) @@ -20,6 +15,7 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) +<<<<<<< HEAD doAssert gState.code.nBl, "Empty file or preprocessor error" if gState.mode == "c": @@ -45,6 +41,18 @@ proc process(gState: State, path: string, astTable: AstTable) = ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code +======= + withCodeAst(gState.code, gState.mode): + if gState.past: + gecho gState.printLisp(root) + elif gState.pnim: + if Feature.ast2 in gState.feature: + ast2.printNim(gState, path, root) + else: + ast.printNim(gState, path, root, astTable) + elif gState.preprocess: + gecho gState.code +>>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim new file mode 100644 index 0000000..f234bc0 --- /dev/null +++ b/nimterop/tshelp.nim @@ -0,0 +1,30 @@ +template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + + # This section is needed to be able to reference + # mode in strformat calls + let + code = inputCode + mode {.inject.} = inputMode + + var parser = tsParserNew() + defer: + parser.tsParserDelete() + + doAssert code.nBl, "Empty code or preprocessor error" + + if mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {mode}" + + var + tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) + root {.inject.} = tree.tsTreeRootNode() + + body + + defer: + tree.tsTreeDelete() \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index 0b3762a..e82430b 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -12,7 +12,6 @@ static: const path = currentSourcePath.parentDir() / "include" / "tast2.h" - when defined(HEADER): cDefine("HEADER") const From 8a0f7954ae3754258706796ccd018f79ddb0158d Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 20:20:34 -0600 Subject: [PATCH 076/255] nimState -> gState --- nimterop/ast2.nim | 8 +- nimterop/comphelp.nim | 4 +- nimterop/exprparser.nim | 247 ++++++++++++++++++++-------------------- nimterop/globals.nim | 9 +- nimterop/toast.nim | 32 +----- 5 files changed, 136 insertions(+), 164 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 1b7fb3d..4fec237 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -104,9 +104,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, nskConst) + result = gState.getOverrideOrSkip(node, origname, nskConst) elif valident.kind != nkNone: - if nimState.addNewIdentifer(name): + if gState.addNewIdentifer(name): # const X* = Y # # nkConstDef( @@ -1400,9 +1400,9 @@ proc addEnum(gState: State, node: TSNode) = for (fname, fval, cexprNode) in fvalSections: var fval = fval if cexprNode.isSome: - fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] # Add other names if node.getName() == "type_definition" and node.len > 1: diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim index 025256a..1709f8b 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/comphelp.nim @@ -7,12 +7,12 @@ proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) if msg < warnMin: raise newException(Exception, msgKindToString(msg)) -proc parseString*(nimState: NimState, str: string): PNode = +proc parseString*(gState: State, str: string): PNode = # Parse a string into Nim AST - use custom error handler that raises # an exception rather than exiting on failure try: result = parseString( - str, nimState.identCache, nimState.config, errorHandler = handleError + str, gState.identCache, gState.config, errorHandler = handleError ) except: decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 2ce4989..047a909 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -30,28 +30,12 @@ import "."/[globals, getters, comphelp, tshelp] # for where Nim can't (such as uint + -int) type - ExprParser* = ref object - state*: NimState - code*: string - name*: string - ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = - ExprParser(state: state, code: code, name: name) - -template techo(msg: varargs[string, `$`]) = - block: - let nimState {.inject.} = exprParser.state - decho join(msg, "") - template val(node: TSNode): string = - exprParser.code.getNodeVal(node) + gState.currentExpr.getNodeVal(node) -proc mode(exprParser: ExprParser): string = - exprParser.state.gState.mode - -proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank @@ -59,19 +43,19 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent var ident = identName if ident != "_": # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, kind, parent) + ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: - result = exprParser.state.getIdent(ident) - elif ident.nBl and ident in exprParser.state.constIdentifiers: - if exprParser.name.nBl: - ident = ident & "." & exprParser.name - result = exprParser.state.getIdent(ident) + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) -proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank - exprParser.getIdent(node.val, kind, parent) + gState.getExprIdent(node.val, kind, parent) proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed @@ -125,53 +109,60 @@ proc getCharLit(charStr: string): PNode {.inline.} = result = newNode(nkCharLit) result.intVal = parseChar(charStr).int64 +proc getFloatNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim float node from a C float expression + suffix + let floatSuffix = number[number.len-1] + try: + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + result = newFloatNode(nkFloatLit, parseFloat(number)) + except ValueError: + raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + +proc getIntNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim int node from a C integer expression + suffix + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) + + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode - result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[number.len-1] - try: - case floatSuffix - of 'l', 'L': - # TODO: handle long double (128 bits) - # result = newNode(nkFloat128Lit) - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - of 'f', 'F': - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - else: - result = newFloatNode(nkFloatLit, parseFloat(number)) - except ValueError: - raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + getFloatNode(number, suffix) else: - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) - else: - result = newNode(nkIntLit) + getIntNode(number, suffix) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) - -proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -189,17 +180,17 @@ proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( - exprParser.state.getIdent("-"), + gState.getIdent("-"), result ) else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processCharacterLiteral(gState: State, node: TSNode): PNode = let val = node.val result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processStringLiteral(gState: State, node: TSNode): PNode = let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -215,9 +206,9 @@ proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = result = newStrNode(nkStrLit, nimStr) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -227,24 +218,24 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va case shiftSym of "<<": - result.add exprParser.state.getIdent("shl") + result.add gState.getIdent("shl") of ">>": - result.add exprParser.state.getIdent("shr") + result.add gState.getIdent("shr") else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let leftNode = exprParser.processTSNode(left, typeofNode) + let leftNode = gState.processTSNode(left, typeofNode) # If the typeofNode is nil, set it # to be the leftNode because C's type coercion # happens left to right, and we want to emulate it if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -252,15 +243,15 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va rightNode ) -proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i], typeofNode)) + result.add(gState.processTSNode(node[i], typeofNode)) -proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( - exprParser.processTSNode(node[0], typeofNode), - exprParser.processTSNode(node[1], typeofNode) + gState.processTSNode(node[0], typeofNode), + gState.processTSNode(node[1], typeofNode) ) proc getNimUnarySym(csymbol: string): string = @@ -292,7 +283,7 @@ proc getNimBinarySym(csymbol: string): string = else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") -proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) result = newNode(nkInfix) @@ -302,16 +293,16 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v binarySym = node.tsNodeChild(1).val.strip() nimSym = getNimBinarySym(binarySym) - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) + result.add gState.getIdent(nimSym) + let leftNode = gState.processTSNode(left, typeofNode) if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -319,7 +310,7 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let @@ -333,31 +324,31 @@ proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: va # Might be bad because we are overwriting the type # There's probably a better way of doing this if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") + typeofNode = gState.getIdent("int64") result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), + gState.getIdent(unarySym), nkPar.newTree( nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) + gState.getIdent("int64"), + gState.processTSNode(child, typeofNode) ) ) ) else: result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) + gState.getIdent(nimSym), + gState.processTSNode(child, typeofNode) ) -proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(exprParser, node, typeofNode) + let binExpr = processBinaryExpression(gState, node, typeofNode) # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, @@ -366,37 +357,37 @@ proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeof elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = processUnaryExpression(exprParser, node, typeofNode) + result = processUnaryExpression(gState, node, typeofNode) else: raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") -proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( - exprParser.state.getIdent("sizeof"), - exprParser.processTSNode(node[0], typeofNode) + gState.getIdent("sizeof"), + gState.processTSNode(node[0], typeofNode) ) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() - techo "NODE: ", nodeName, ", VAL: ", node.val + decho "NODE: ", nodeName, ", VAL: ", node.val case nodeName of "number_literal": # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 - result = exprParser.processNumberLiteral(node) + result = gState.processNumberLiteral(node) of "string_literal": # Input -> "foo\0\x42" # Output -> "foo\0" - result = exprParser.processStringLiteral(node) + result = gState.processStringLiteral(node) of "char_literal": # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences # Output -> - result = exprParser.processCharacterLiteral(node) + result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain # an ERROR node. If that's the case, they usually contain children with @@ -405,21 +396,21 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ if node.len == 1: - result = exprParser.processTSNode(node[0], typeofNode) + result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: result = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add exprParser.processTSNode(node[i], typeofNode) + result.add gState.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": # Input -> (IDENT - OTHERIDENT) # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) - result = exprParser.processParenthesizedExpr(node, typeofNode) + result = gState.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": # Input -> sizeof(char) # Output -> sizeof(cchar) - result = exprParser.processSizeofExpression(node, typeofNode) + result = gState.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", @@ -434,55 +425,61 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) - result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) + result = gState.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": # Input -> a >> b, a << b # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = exprParser.processShiftExpression(node, typeofNode) + result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) - result = exprParser.processCastExpression(node, typeofNode) + result = gState.processCastExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": # Input -> true, false # Output -> true, false - result = exprParser.state.parseString(node.val) + result = gState.parseString(node.val) of "type_descriptor", "sized_type_specifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc let ty = getType(node.val) if ty.len > 0: # If ty is not empty, one of C's builtin types has been found - result = exprParser.getIdent(ty, nskType, parent=node.getName()) + result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: - result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + result = gState.getExprIdent(node.val, nskType, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) - result = exprParser.getIdent(node, parent=node.getName()) + result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RESULT: ", result + decho "NODE RESULT: ", result -proc parseCExpression*(state: NimState, code: string, name = ""): PNode = +proc parseCExpression*(gState: State, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil - let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser.code, exprParser.mode): - result = exprParser.processTSNode(root, tnode) + withCodeAst(gState.currentExpr, gState.mode): + result = gState.processTSNode(root, tnode) except ExprParseError as e: - techo e.msg + decho e.msg result = newNode(nkNone) except Exception as e: - techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + decho "UNEXPECTED EXCEPTION: ", e.msg + result = newNode(nkNone) + + # Clear the state + gState.currentExpr = "" + gState.currentTyCastName = "" \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index b964da9..0d0b4cd 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,4 +1,4 @@ -import sequtils, sets, tables +import sequtils, sets, tables, strutils import regex @@ -93,6 +93,9 @@ type currentHeader*, impShort*, sourceFile*: string + # Used for the exprparser.nim module + currentExpr*, currentTyCastName*: string + data*: seq[tuple[name, val: string]] nodeBranch*: seq[string] @@ -119,6 +122,6 @@ when not declared(CIMPORT): else: gState.outputHandle.writeLine(args) - template decho*(str: untyped): untyped = + template decho*(args: varargs[string, `$`]): untyped = if gState.debug: - gecho str.getCommented() + gecho join(args, "").getCommented() \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index afb511d..98045bf 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -15,44 +15,16 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) -<<<<<<< HEAD - doAssert gState.code.nBl, "Empty file or preprocessor error" - - if gState.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif gState.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {gState.mode}" - - var - tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32) - root = tree.tsTreeRootNode() - - defer: - tree.tsTreeDelete() - - if gState.past: - gecho gState.printLisp(root) - elif gState.pnim: - if Feature.ast2 in gState.feature: - ast2.parseNim(gState, path, root) - else: - ast.parseNim(gState, path, root, astTable) - elif gState.preprocess: - gecho gState.code -======= withCodeAst(gState.code, gState.mode): if gState.past: gecho gState.printLisp(root) elif gState.pnim: if Feature.ast2 in gState.feature: - ast2.printNim(gState, path, root) + ast2.parseNim(gState, path, root) else: - ast.printNim(gState, path, root, astTable) + ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code ->>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( From c0977fa8271f462c20a29f4bb0976c638a534b45 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:15:40 -0600 Subject: [PATCH 077/255] Update code with comments and add debug expr proc --- nimterop/exprparser.nim | 143 +++++++++++++++++++++++++++++++++++++++- nimterop/getters.nim | 8 +-- 2 files changed, 144 insertions(+), 7 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 047a909..7acb375 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -35,6 +35,11 @@ type template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) +proc printDebugExpr*(gState: State, node: TSNode) = + if gState.debug: + gecho ("Input => " & node.val).getCommented() + gecho gState.currentExpr.printLisp(node).getCommented() + proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## @@ -187,10 +192,29 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral(gState: State, node: TSNode): PNode = + # Input => 'G' + # + # (char_literal 1 1 3 "'G'") + # + # Output => 'G' + # + # nkCharLit("G") let val = node.val result = getCharLit(val[1 ..< val.len - 1]) proc processStringLiteral(gState: State, node: TSNode): PNode = + # Input => "\n\rfoobar\0\'" + # + # (string_literal 1 1 16 ""\n\rfoobar\0\'"" + # (escape_sequence 1 2 2 "\n") + # (escape_sequence 1 4 2 "\r") + # (escape_sequence 1 12 2 "\0") + # (escape_sequence 1 14 2 "\'") + # ) + # + # Output => "\n\cfoobar\x00\'" + # + # nkStrLit("\x0A\x0Dfoobar\x00\'") let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -209,6 +233,26 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => a >> b + # + # (shift_expression 1 2 6 + # (identifier 1 2 1 "a") + # (identifier 1 7 1 "b") + # ) + # + # Output => a shr typeof(a)(b) + # + # nkInfix( + # nkIdent("shr"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let left = node[0] @@ -244,11 +288,56 @@ proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (a + b) + # + # (parenthesized_expression 1 1 7 + # (math_expression 1 2 5 + # (identifier 1 2 1 "a") + # (identifier 1 6 1 "b") + # ) + # ) + # + # Output => (typeof(a)(a + typeof(a)(b))) + # + # nkPar( + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkInfix( + # nkIdent("+"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) + # ) + # ) result = newNode(nkPar) for i in 0 ..< node.len(): result.add(gState.processTSNode(node[i], typeofNode)) proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (int)a + # + # (cast_expression 1 1 6 "(int)a" + # (type_descriptor 1 2 3 "int" + # (primitive_type 1 2 3 "int") + # ) + # (identifier 1 6 1 "a") + # ) + # + # Output => cast[cint](a) + # + # nkCast( + # nkIdent("cint"), + # nkIdent("a") + # ) result = nkCast.newTree( gState.processTSNode(node[0], typeofNode), gState.processTSNode(node[1], typeofNode) @@ -285,6 +374,27 @@ proc getNimBinarySym(csymbol: string): string = proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) + # + # Input => a == b + # + # (equality_expression 1 1 6 + # (identifier 1 1 1 "a") + # (identifier 1 6 1 "b") + # ) + # + # Output => a == typeof(a)(b) + # + # nkInfix( + # nkIdent("=="), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let @@ -311,6 +421,20 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => !a + # + # (logical_expression 1 1 2 "!a" + # (identifier 1 2 1 "a") + # ) + # + # Output => (not a) + # + # nkPar( + # nkPrefix( + # nkIdent("not"), + # nkIdent("a") + # ) + # ) result = newNode(nkPar) let @@ -342,6 +466,7 @@ proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) @@ -362,6 +487,20 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => sizeof(int) + # + # (sizeof_expression 1 1 11 "sizeof(int)" + # (type_descriptor 1 8 3 "int" + # (primitive_type 1 8 3 "int") + # ) + # ) + # + # Output => sizeof(cint) + # + # nkCall( + # nkIdent("sizeof"), + # nkIdent("cint") + # ) result = nkCall.newTree( gState.getIdent("sizeof"), gState.processTSNode(node[0], typeofNode) @@ -385,8 +524,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Output -> "foo\0" result = gState.processStringLiteral(node) of "char_literal": - # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences - # Output -> + # Input -> 'F', '\060' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> 'F', '0', 'Z', '\r' result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b7744ce..121c8d5 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -436,13 +436,13 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: - result &= "'" & pnode.intVal.char & "')" + result &= ($pnode.intVal.char).escape & ")" of nkIntLit..nkUInt64Lit: result &= $pnode.intVal & ")" of nkFloatLit..nkFloat128Lit: result &= $pnode.floatVal & ")" of nkStrLit..nkTripleStrLit: - result &= "\"" & pnode.strVal & "\")" + result &= pnode.strVal.escape & ")" of nkSym: result &= $pnode.sym & ")" of nkIdent: @@ -460,16 +460,14 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(gState: State, node: TSNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug: gecho ("Input => " & gState.getNodeVal(node)).getCommented() gecho gState.printLisp(node).getCommented() proc printDebug*(gState: State, pnode: PNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug and pnode.kind != nkNone: gecho ("Output => " & $pnode).getCommented() - gecho gState.printTree(pnode).getCommented() + gecho gState.printTree(pnode) # Compiler shortcuts From 2813dc61f22aecaaaa82054f144a48c4de520cf2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:45:19 -0600 Subject: [PATCH 078/255] Consolidate shift_expression into binary_expression and write more tests --- nimterop/exprparser.nim | 91 +++++++++-------------------------------- tests/include/tast2.h | 10 +++++ tests/tast2.nim | 14 ++++++- 3 files changed, 42 insertions(+), 73 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 7acb375..a59a546 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -232,61 +232,6 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = - # Input => a >> b - # - # (shift_expression 1 2 6 - # (identifier 1 2 1 "a") - # (identifier 1 7 1 "b") - # ) - # - # Output => a shr typeof(a)(b) - # - # nkInfix( - # nkIdent("shr"), - # nkIdent("a"), - # nkCall( - # nkCall( - # nkIdent("typeof"), - # nkIdent("a") - # ), - # nkIdent("b") - # ) - # ) - result = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let shiftSym = node.tsNodeChild(1).val.strip() - - case shiftSym - of "<<": - result.add gState.getIdent("shl") - of ">>": - result.add gState.getIdent("shr") - else: - raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - - let leftNode = gState.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - gState.getIdent("typeof"), - leftNode - ) - - let rightNode = gState.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => (a + b) # @@ -369,6 +314,10 @@ proc getNimBinarySym(csymbol: string): string = result = csymbol of "%": result = "mod" + of "<<": + result = "shl" + of ">>": + result = "shr" else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") @@ -419,6 +368,15 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) typeofNode, rightNode ) + if binarySym == "/": + # Special case. Nim's operators generally output + # the same type they take in, except for division. + # So we need to emulate C here and cast the whole + # expression to the type of the first arg + result = nkCall.newTree( + typeofNode, + result + ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => !a @@ -469,17 +427,7 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) - - # Make sure the statement is of the same type as the left - # hand argument, since some expressions return a differing - # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(gState, node, typeofNode) - # Note that this temp var binExpr is needed for some reason, or else we get a segfault - result = nkCall.newTree( - typeofNode, - binexpr - ) - + result = processBinaryExpression(gState, node, typeofNode) elif node.len() == 1: # Node has only one child, ie -(20 + 7) result = processUnaryExpression(gState, node, typeofNode) @@ -553,8 +501,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", - "bitwise_expression", "equality_expression", "binary_expression": - # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + "bitwise_expression", "equality_expression", "binary_expression", + "shift_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b # Output -> # typeof(a)(a == typeof(a)(b)) # typeof(a)(a != typeof(a)(b)) @@ -564,11 +513,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) + # a shr typeof(a)(b) + # a shl typeof(a)(b) result = gState.processUnaryOrBinaryExpression(node, typeofNode) - of "shift_expression": - # Input -> a >> b, a << b - # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 53b4d64..e72a3b8 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,6 +24,16 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 +#define a 100 +#define b 200 +#define EQ1 a <= b +#define EQ2 a >= b +#define EQ3 a > b +#define EQ4 a < b +#define EQ5 a != b +#define EQ6 a == b + +#define SIZEOF sizeof(char) #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index e82430b..57bfc90 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,6 +115,18 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) +assert a == 100 +assert b == 200 + +assert EQ1 == (a <= b) +assert EQ2 == (a >= b) +assert EQ3 == (a > b) +assert EQ4 == (a < b) +assert EQ5 == (a != b) +assert EQ6 == (a == b) + +assert SIZEOF == 1 + assert COERCE == 645635670332'u64 assert COERCE2 == 645635670332'i64 @@ -303,7 +315,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[123 + type(123)(132), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 9e799ee3b19ac152d1c2403c6af1e3806e1a31fe Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:53:52 -0600 Subject: [PATCH 079/255] Fix tmath tests Modify tmath.nim to have one skip symbol Fix macro expansion with common value Fix tmath again --- tests/include/tast2.h | 16 ++++++++-------- tests/tast2.nim | 16 ++++++++-------- tests/tmath.nim | 6 ++++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e72a3b8..815791f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,14 +24,14 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 -#define a 100 -#define b 200 -#define EQ1 a <= b -#define EQ2 a >= b -#define EQ3 a > b -#define EQ4 a < b -#define EQ5 a != b -#define EQ6 a == b +#define AVAL 100 +#define BVAL 200 +#define EQ1 AVAL <= BVAL +#define EQ2 AVAL >= BVAL +#define EQ3 AVAL > BVAL +#define EQ4 AVAL < BVAL +#define EQ5 AVAL != BVAL +#define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) diff --git a/tests/tast2.nim b/tests/tast2.nim index 57bfc90..da7fafa 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,15 +115,15 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert a == 100 -assert b == 200 +assert AVAL == 100 +assert BVAL == 200 -assert EQ1 == (a <= b) -assert EQ2 == (a >= b) -assert EQ3 == (a > b) -assert EQ4 == (a < b) -assert EQ5 == (a != b) -assert EQ6 == (a == b) +assert EQ1 == (AVAL <= BVAL) +assert EQ2 == (AVAL >= BVAL) +assert EQ3 == (AVAL > BVAL) +assert EQ4 == (AVAL < BVAL) +assert EQ5 == (AVAL != BVAL) +assert EQ6 == (AVAL == BVAL) assert SIZEOF == 1 diff --git a/tests/tmath.nim b/tests/tmath.nim index bbf7abd..b8477c1 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -14,9 +14,11 @@ when defined(windows): static: when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] + # FP_ILOGB0 and FP_ILOGBNAN are casts that are unsupported + # on lower Nim VMs + cSkipSymbol @["math_errhandling", "FP_ILOGB0", "FP_ILOGBNAN"] else: - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] + cSkipSymbol @["math_errhandling"] cDebug() cDisableCaching() cAddStdDir() From 073dc5d35a70270c49234842bfbb7f3e608c539e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 19:21:57 -0600 Subject: [PATCH 080/255] Add hack for skipping types as root nodes --- nimterop/ast2.nim | 30 ++++++++++++++++++++++++++---- nimterop/exprparser.nim | 21 +++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4fec237..bb41ebe 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,9 +4,9 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/api +import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, exprparser, comphelp] +import "."/[globals, getters, exprparser, comphelp, tshelp] proc getPtrType*(str: string): string = result = case str: @@ -99,8 +99,30 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) - valident = - gState.parseCExpression(val) + var valident = newNode(nkNone) + + withCodeAst(val, gState.mode): + # This section is a hack for determining that the first + # node is a type, which shouldn't be accepted by a const + # def section. Need to replace this with some other mechanism + # to handle type aliases + var maybeTyNode: TSNode + # Take the very first node, which may be 2 levels + # down if there is an error node + if root.len > 0 and root[0].getName() == "ERROR": + maybeTyNode = root[0][0] + elif root.len > 0: + maybeTyNode = root[0] + + if not maybeTyNode.isNil: + let name = maybeTyNode.getName() + case name + of "type_descriptor", "sized_type_specifier": + discard + else: + # Can't do gState.parseCExpression(root) here for some reason? + # get a SEGFAULT if we use root + valident = gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a59a546..6e102ea 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -547,18 +547,15 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = decho "NODE RESULT: ", result -proc parseCExpression*(gState: State, code: string, name = ""): PNode = - ## Convert the C string to a nim PNode tree - gState.currentExpr = code - gState.currentTyCastName = name +proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = + ## Parse a c expression from a root ts node - result = newNode(nkNone) - # This is used for keeping track of the type of the first + # This var is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil + result = newNode(nkNone) try: - withCodeAst(gState.currentExpr, gState.mode): - result = gState.processTSNode(root, tnode) + result = gState.processTSNode(codeRoot, tnode) except ExprParseError as e: decho e.msg result = newNode(nkNone) @@ -566,6 +563,14 @@ proc parseCExpression*(gState: State, code: string, name = ""): PNode = decho "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) +proc parseCExpression*(gState: State, code: string, name = ""): PNode = + ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + + withCodeAst(gState.currentExpr, gState.mode): + result = gState.parseCExpression(root, name) + # Clear the state gState.currentExpr = "" gState.currentTyCastName = "" \ No newline at end of file From cc460b2779fc8b2035c19e0ba8f460cf124d474f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:20:05 -0600 Subject: [PATCH 081/255] Add skippedSymbols for determining if a type has been skipped --- nimterop/ast2.nim | 3 +++ nimterop/exprparser.nim | 25 +++++++++++++------------ nimterop/globals.nim | 3 +++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index bb41ebe..2ae4a70 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -41,6 +41,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS result = pnode[0][0] else: gecho &"\n# $1'{origname}' skipped" % skind + gState.skippedSyms.incl origname if gState.debug: gState.skipStr &= &"\n{gState.getNodeVal(node)}" @@ -99,6 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) + var valident = newNode(nkNone) withCodeAst(val, gState.mode): @@ -151,6 +153,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = gecho &"# const '{origname}' is duplicate, skipped" else: gecho &"# const '{origname}' has invalid value '{val}'" + gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = # Add a const to the AST diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 6e102ea..1f6decc 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,16 +45,17 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - var ident = identName - if ident != "_": - # Process the identifier through cPlugin - ident = gState.getIdentifier(ident, kind, parent) - if kind == nskType: - result = gState.getIdent(ident) - elif ident.nBl and ident in gState.constIdentifiers: - if gState.currentTyCastName.nBl: - ident = ident & "." & gState.currentTyCastName - result = gState.getIdent(ident) + if identName notin gState.skippedSyms: + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = gState.getIdentifier(ident, kind, parent) + if kind == nskType: + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` @@ -534,8 +535,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: result = gState.getExprIdent(node.val, nskType, parent=node.getName()) - if result.kind == nkNone: - raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 0d0b4cd..d433ab5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,6 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] + # All symbols that have been skipped + skippedSyms*: HashSet[string] + # Legacy ast fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From 3e28501826f5d655ad3797cbf29009082be126e1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:59:21 -0600 Subject: [PATCH 082/255] Fix comments breaking code --- nimterop/exprparser.nim | 14 ++++++++++++-- tests/include/tast2.h | 4 ++-- tests/tpcre.nim | 1 - 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f6decc..e9ee502 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,9 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - result = newNode(nkStmtListExpr) + let res = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add gState.processTSNode(node[i], typeofNode) + let node = gState.processTSNode(node[i], typeofNode) + if node.kind != nkNone: + res.add node + if res.len == 1: + result = res[0] + elif res.len > 1: + result = res + else: + result = newNode(nkNone) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": @@ -543,6 +551,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") + of "comment": + discard else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 815791f..42c852f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,8 +35,8 @@ extern "C" { #define SIZEOF sizeof(char) -#define NULLCHAR '\0' -#define OCTCHAR '\012' +#define NULLCHAR '\0'/* comments should not break things*/ +#define OCTCHAR '\012' // nor should this comment #define HEXCHAR '\xFE' #define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 3bbd1ba..c8e8059 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,7 +7,6 @@ const pcreH = baseDir/"pcre.h.in" static: - cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From 4e687dd80783e8ff7bca9ba8ba6ba61d3a933fc2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 21:52:34 -0600 Subject: [PATCH 083/255] Don't support multiple base nodes yet --- nimterop/exprparser.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e9ee502..c74f0b6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,17 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - let res = newNode(nkStmtListExpr) + var nodes: seq[PNode] for i in 0 ..< node.len: - let node = gState.processTSNode(node[i], typeofNode) - if node.kind != nkNone: - res.add node - if res.len == 1: - result = res[0] - elif res.len > 1: - result = res - else: - result = newNode(nkNone) + let subNode = gState.processTSNode(node[i], typeofNode) + if subNode.kind != nkNone: + nodes.add(subNode) + # Multiple nodes can get tricky. Don't support them yet, unless they + # have at most one valid node + if nodes.len > 1: + raise newException(ExprParseError, &"Node type \"{nodeName}\" with val ({node.val}) has more than one non empty node") + if nodes.len == 1: + result = nodes[0] else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": From 322a0031984cefc278d2a377daa27bffdcbcae73 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 22:11:39 -0600 Subject: [PATCH 084/255] Add test for not supported nodes --- tests/include/tast2.h | 2 ++ tests/tast2.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 42c852f..b47a801 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -34,6 +34,8 @@ extern "C" { #define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) +#define REG_STR "regular string" +#define NOTSUPPORTEDSTR "not a " REG_STR #define NULLCHAR '\0'/* comments should not break things*/ #define OCTCHAR '\012' // nor should this comment diff --git a/tests/tast2.nim b/tests/tast2.nim index da7fafa..4cfbeac 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -109,6 +109,8 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert not defined(NOTSUPPORTEDSTR) + assert UEXPR == (1234.uint shl 1) assert ULEXPR == (1234.uint32 shl 2) assert ULLEXPR == (1234.uint64 shl 3) From 89c10c4b25226a88062d01e6bf57a9e9146920c5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 14:25:42 -0600 Subject: [PATCH 085/255] Address some PR comments --- nimterop/ast2.nim | 12 +++++++----- nimterop/globals.nim | 4 +++- nimterop/tshelp.nim | 14 ++++++-------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2ae4a70..474ec69 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,7 +4,7 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/[api, c, cpp] +import "."/treesitter/api import "."/[globals, getters, exprparser, comphelp, tshelp] @@ -1391,7 +1391,9 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] - fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + # Hold all of field information so that we can add all of them + # after the const identifiers has been updated + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1410,9 +1412,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fvalSections.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]))) else: - fvalSections.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode))) fnames.incl fname prev = fname @@ -1422,7 +1424,7 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fvalSections: + for (fname, fval, cexprNode) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d433ab5..5db17a3 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,7 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] - # All symbols that have been skipped + # All symbols that have been skipped due to + # being unwrappable or the user provided + # override is blank skippedSyms*: HashSet[string] # Legacy ast fields, remove when ast2 becomes default diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index f234bc0..109321c 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,11 +1,9 @@ -template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code +import "."/treesitter/[c, cpp] - # This section is needed to be able to reference - # mode in strformat calls - let - code = inputCode - mode {.inject.} = inputMode +template withCodeAst*(code: string, mode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + mixin treeSitterC + mixin treeSitterCpp var parser = tsParserNew() defer: @@ -18,7 +16,7 @@ template withCodeAst*(inputCode: string, inputMode: string, body: untyped): unty elif mode == "cpp": doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" else: - doAssert false, &"Invalid parser {mode}" + doAssert false, "Invalid parser " & mode var tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) From 33d77f82e878c86c0edf3bd21d359e5f40492ad2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 27 Apr 2020 22:38:49 -0500 Subject: [PATCH 086/255] Fix #151 and fix #153 - detect nim.cfg --- README.md | 16 ++-- nimterop/build.nim | 50 +++++------ nimterop/cimport.nim | 1 + nimterop/docs.nim | 5 +- nimterop/nimconf.nim | 199 +++++++++++++++++++++++++++++++++++++++++++ tests/getheader.nims | 1 + 6 files changed, 238 insertions(+), 34 deletions(-) create mode 100644 nimterop/nimconf.nim diff --git a/README.md b/README.md index 0036187..53b37e7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The nimterop wrapping functionality is still limited to C but is constantly expa Nimterop has seen some adoption within the community and the simplicity and success of this approach justifies additional investment of time and effort. Regardless, the goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation. -### Installation +## Installation Nimterop can be installed via [Nimble](https://github.com/nim-lang/nimble): @@ -28,11 +28,11 @@ nimble build This will download and install nimterop in the standard Nimble package location, typically `~/.nimble`. Once installed, it can be imported into any Nim program. -### Usage +## Usage Detailed documentation can be found [here](https://nimterop.github.io/nimterop/theindex.html). Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work. -#### Build API +### Build API Creating a wrapper has two parts, the first is to setup the C library. This includes downloading it or finding it if already installed, and building it if applicable. The `getHeader()` high-level API provides all of this functionality as a convenience. Following is an example of using the high-level `getHeader()` API to perform all building and linking automatically: @@ -103,7 +103,7 @@ The `-d:xxxYYY` Nim define flags have already been described above and can be sp If more fine-tuned control is desired over the build process, it is possible to manually control all steps that `getHeader()` performs by directly using the API provided by [build](https://nimterop.github.io/nimterop/build.html). Note also that there is no requirement to use these APIs to setup the library. Any other established mechanisms can be used to do so any limitations imposed by Nimterop are unintentional and feedback is most welcome. -#### Wrapper API +### Wrapper API Once the C library is setup, the next step is to create wrappers that inform Nim of all the types and functions that are available. Following is a simple example covering the API: @@ -152,7 +152,7 @@ __Compiling source__ The job of building and compiling the underlying C library is best left to the build mechanism selected by the library author so using `getHeader()` is recommended. For simpler projects with a few `.c` files though, `cCompile()` should be more than enough. It is not recommended for larger projects which heavily rely on functionality offered by build tools. Recreating reliable logic in Nim can be tedious and one can expect minimal support from that author if their tested build mechanism is not used. -#### Command line API +### Command line API The `toast` binary can also be used directly on the CLI, similar to `c2nim`. The `cPlugin()` interface @@ -190,7 +190,7 @@ Options: -O=, --symOverride= strings {} skip generating specified symbols ``` -### Why nimterop +## Why nimterop Nim has one of the best FFI you can find - importing C/C++ is supported out of the box. All you need to provide is type and proc definitions for Nim to interop with C/C++ binaries. Generation of these wrappers is easy for simple libraries but can quickly get out of hand. [c2nim](https://github.com/nim-lang/c2nim) greatly helps here by parsing and converting C/C++ into Nim but is limited due to the complex and constantly evolving C/C++ grammar. [nimgen](https://github.com/genotrance/nimgen) mainly focused on automating the wrapping process with `c2nim` and filled some holes but is again limited to `c2nim` capabilities. @@ -212,10 +212,10 @@ The con of this approach of delegating to the preprocessor is that the Nim wrapp This is part of the reason why Nimterop provides a wrapper API so that the generation of wrappers is Nim code that can be rendered as part of the build process on the target platform. It helps to think of Nimterop as a build time tool like `cmake` that renders artifacts on the target rather than a tool whose generated artifacts should be checked into source control. Regardless, both the wrapper API and the `toast` command line still allow saving the wrapper output to a file to be stored in source control since it might work well enough for many projects. -__Credits__ +## Credits Nimterop depends on [tree-sitter](http://tree-sitter.github.io/tree-sitter/) and all licensing terms of [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/master/LICENSE) apply to the usage of this package. The tree-sitter functionality is pulled and wrapped using nimterop itself. -__Feedback__ +## Feedback Nimterop is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/nimterop/nimterop) with an MIT license so issues, forks and PRs are most appreciated. diff --git a/nimterop/build.nim b/nimterop/build.nim index a025dc0..e11cd7b 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,11 +4,32 @@ import os except findExe, sleep import regex +proc fixCmd(cmd: string): string = + when defined(Windows): + # Replace 'cd d:\abc' with 'd: && cd d:\abc` + var filteredCmd = cmd + if cmd.toLower().startsWith("cd"): + var + colonIndex = cmd.find(":") + driveLetter = cmd.substr(colonIndex-1, colonIndex) + if (driveLetter[0].isAlphaAscii() and + driveLetter[1] == ':' and + colonIndex == 4): + filteredCmd = &"{driveLetter} && {cmd}" + result = "cmd /c " & filteredCmd + elif defined(posix): + result = cmd + else: + doAssert false + proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: result = result.quoteShell +# Nim cfg file related functionality +include "."/nimconf + proc sleep*(milsecs: int) = ## Sleep at compile time let @@ -20,14 +41,9 @@ proc sleep*(milsecs: int) = discard gorgeEx(cmd & $(milsecs / 1000)) -proc getOsCacheDir(): string = - when defined(posix): - result = getEnv("XDG_CACHE_HOME", getHomeDir() / ".cache") / "nim" - else: - result = getHomeDir() / "nimcache" - proc getNimteropCacheDir(): string = - result = getOsCacheDir() / "nimterop" + # Get location to cache all nimterop artifacts + result = getNimcacheDir() / "nimterop" proc getCurrentNimCompiler*(): string = result = getCurrentCompilerExe() @@ -46,24 +62,8 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, ## `die = false` - return on errors ## `cache = true` - cache results unless cleared with -f ## `cacheKey` - key to create unique cache entry - var - ccmd = "" - when defined(Windows): - # Replace 'cd d:\abc' with 'd: && cd d:\abc` - var filteredCmd = cmd - if cmd.toLower().startsWith("cd"): - var - colonIndex = cmd.find(":") - driveLetter = cmd.substr(colonIndex-1, colonIndex) - if (driveLetter[0].isAlphaAscii() and - driveLetter[1] == ':' and - colonIndex == 4): - filteredCmd = &"{driveLetter} && {cmd}" - ccmd = "cmd /c " & filteredCmd - elif defined(posix): - ccmd = cmd - else: - doAssert false + let + ccmd = fixCmd(cmd) when nimvm: # Cache results for speedup if cache = true diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 3e58bc8..4afb27c 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -286,6 +286,7 @@ proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") if not fileExists(path) or gStateCT.nocache or compileOption("forceBuild"): mkDir(path.parentDir()) writeFile(path, data) + writeNimConfig(path & ".cfg") doAssert fileExists(path), "Unable to write plugin file: " & path diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 97fab96..6eb812b 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -2,8 +2,11 @@ import macros, strformat from os import parentDir, getCurrentCompilerExe, DirSep +when defined(nimdoc) or (NimMajor, NimMinor) >= (1, 3): + from os import paramCount, paramStr + when defined(nimdoc): - from os import getCurrentDir, paramCount, paramStr + from os import getCurrentDir proc getNimRootDir(): string = #[ diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim new file mode 100644 index 0000000..fdf2648 --- /dev/null +++ b/nimterop/nimconf.nim @@ -0,0 +1,199 @@ +import json, macros, os, osproc, sets, strformat, strutils + +when nimvm: + when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + import std/compilesettings +else: + discard + +# Config detected with std/compilesettings or `nim dump` +type + Config* = ref object + NimMajor*: int + NimMinor*: int + NimPatch*: int + + paths*: OrderedSet[string] + nimblePaths*: OrderedSet[string] + nimcacheDir*: string + +proc getJson(projectDir: string): JsonNode = + # Get `nim dump` json value for `projectDir` + var + cmd = "nim --hints:off --dump.format:json dump dummy" + dump = "" + ret = 0 + + if projectDir.len != 0: + # Run `nim dump` in `projectDir` if specified + cmd = &"cd {projectDir.sanitizePath} && " & cmd + + cmd = fixCmd(cmd) + when nimvm: + (dump, ret) = gorgeEx(cmd) + else: + (dump, ret) = execCmdEx(cmd) + + try: + result = parseJson(dump) + except JsonParsingError as e: + echo "# Failed to parse `nim dump` output: " & e.msg + +proc getOsCacheDir(): string = + # OS default cache directory + when defined(posix): + result = getEnv("XDG_CACHE_HOME", getHomeDir() / ".cache") / "nim" + else: + result = getHomeDir() / "nimcache" + +proc getProjectDir*(): string = + ## Get project directory for this compilation - returns `""` at runtime + when nimvm: + when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + # If nim v1.2.0+, get from `std/compilesettings` + result = querySetting(projectFull).parentDir() + else: + # Get from `macros` + result = getProjectPath() + else: + discard + +proc getNimcacheDir*(projectDir = ""): string = + ## Get nimcache directory for current compilation or specified `projectDir` + when nimvm: + when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + # Get value at compile time from `std/compilesettings` + result = querySetting(SingleValueSetting.nimcacheDir) + else: + discard + + # Not Nim v1.2.0+ or runtime + if result.len == 0: + let + # Get project directory for < v1.2.0 at compile time + projectDir = if projectDir.len != 0: projectDir else: getProjectDir() + + # Use `nim dump` to figure out nimcache for `projectDir` + let + dumpJson = getJson(projectDir) + + if dumpJson != nil and dumpJson.hasKey("nimcache"): + result = dumpJson["nimcache"].getStr() + let + (head, tail) = result.splitPath() + if "dummy" in tail: + # Remove `dummy_d` subdir when default nimcache + result = head + + # Set to OS defaults if not detectable + if result.len == 0: + result = getOsCacheDir() + +proc jsonToSeq(node: JsonNode, key: string): seq[string] = + # Convert JsonArray to seq[string] for specified `key` + if node.hasKey(key): + for elem in node[key].getElems(): + result.add elem.getStr() + +proc getNimConfig*(projectDir = ""): Config = + # Get `paths` - list of paths to be forwarded to Nim + result = new(Config) + var + libPath, version: string + lazyPaths, searchPaths: seq[string] + + when nimvm: + result.NimMajor = NimMajor + result.NimMinor = NimMinor + result.NimPatch = NimPatch + + when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + # Get value at compile time from `std/compilesettings` + libPath = getCurrentCompilerExe().parentDir().parentDir() / "lib" + lazyPaths = querySettingSeq(MultipleValueSetting.lazyPaths) + searchPaths = querySettingSeq(MultipleValueSetting.searchPaths) + result.nimcacheDir = querySetting(SingleValueSetting.nimcacheDir) + else: + discard + + let + # Get project directory for < v1.2.0 at compile time + projectDir = if projectDir.len != 0: projectDir else: getProjectDir() + + # Not Nim v1.2.0+ or runtime + if libPath.len == 0: + let + dumpJson = getJson(projectDir) + + if dumpJson != nil: + if dumpJson.hasKey("version"): + version = dumpJson["version"].getStr() + lazyPaths = jsonToSeq(dumpJson, "lazyPaths") + searchPaths = jsonToSeq(dumpJson, "lib_paths") + if dumpJson.hasKey("libpath"): + libPath = dumpJson["libpath"].getStr() + elif searchPaths.len != 0: + # Usually `libPath` is last entry in `searchPaths` + libPath = searchPaths[^1] + + # Parse version + if version.len != 0: + let + splversion = version.split({'.'}, maxsplit = 3) + result.NimMajor = splversion[0].parseInt() + result.NimMinor = splversion[1].parseInt() + result.NimPatch = splversion[2].parseInt() + + # Find non standard lib paths added to `searchPath` + for path in searchPaths: + if libPath notin path: + result.paths.incl path + + # Find `nimblePaths` in `lazyPaths` + for path in lazyPaths: + let + (_, tail) = path.strip(leading = false, chars = {'/', '\\'}).splitPath() + if tail == "pkgs": + # Nimble path probably + result.nimblePaths.incl path + + # Find `paths` in `lazyPaths` that aren't within `nimblePaths` + # Have to do this separately since `nimblePaths` could be after + # packages in `lazyPaths` + for path in lazyPaths: + var skip = false + for npath in result.nimblePaths: + if npath in path: + skip = true + break + if not skip: + result.paths.incl path + + result.nimcacheDir = getNimcacheDir(projectDir) + +proc writeNimConfig*(cfg: Config, cfgFile: string) = + # Write Nim configuration to file + var + cfgOut = &"--nimcache:\"{cfg.nimcacheDir}\"\n" + + if (cfg.NimMajor, cfg.NimMinor, cfg.NimPatch) >= (1, 2, 0): + # --clearNimbleCache if Nim v1.2.0+ + cfgOut &= "--clearNimblePath\n" + + # Add `nimblePaths` if detected - v1.2.0+ + for path in cfg.nimblePaths: + cfgOut &= &"--nimblePath:\"{path}\"\n" + + # Add `paths` in all cases if any detected + for path in cfg.paths: + cfgOut &= &"--path:\"{path}\"\n" + + when defined(windows): + cfgOut = cfgOut.replace("\\", "/") + + writeFile(cfgFile, cfgOut) + +proc writeNimConfig*(cfgFile: string, projectDir = "") = + let + cfg = getNimConfig(projectDir) + writeNimConfig(cfg, cfgFile) diff --git a/tests/getheader.nims b/tests/getheader.nims index 7a0fc80..ef501ab 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -11,6 +11,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = if not delete: ccmd = ccmd.replace(" -f ", " ") + echo ccmd var (outp, exitC) = gorgeEx(ccmd) echo outp From c57666d5dbafaf88c4a8b53149ce9b07f9391242 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Apr 2020 10:30:24 -0500 Subject: [PATCH 087/255] Enable conf detection in toast standalone --- nimterop/getters.nim | 8 +++++++- nimterop/nimconf.nim | 36 ++++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 121c8d5..d829cf0 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -726,7 +726,13 @@ proc loadPlugin*(gState: State, sourcePath: string) = pdll = sourcePath.dll if not fileExists(pdll) or sourcePath.getLastModificationTime() > pdll.getLastModificationTime(): - discard execAction(&"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {sourcePath.sanitizePath}") + let + # Get Nim configuration flags if not already specified in a .cfg file + flags = + if fileExists(sourcePath & ".cfg"): "" + else: getNimConfigFlags(getCurrentDir()) + cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {sourcePath.sanitizePath}" + discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath let lib = loadLib(pdll) diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index fdf2648..664675e 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -171,29 +171,41 @@ proc getNimConfig*(projectDir = ""): Config = result.nimcacheDir = getNimcacheDir(projectDir) -proc writeNimConfig*(cfg: Config, cfgFile: string) = - # Write Nim configuration to file - var - cfgOut = &"--nimcache:\"{cfg.nimcacheDir}\"\n" +proc getNimConfigFlags(cfg: Config): string = + # Convert configuration into Nim flags for cfg file or command line + result = &"--nimcache:\"{cfg.nimcacheDir}\"\n" if (cfg.NimMajor, cfg.NimMinor, cfg.NimPatch) >= (1, 2, 0): # --clearNimbleCache if Nim v1.2.0+ - cfgOut &= "--clearNimblePath\n" + result &= "--clearNimblePath\n" # Add `nimblePaths` if detected - v1.2.0+ for path in cfg.nimblePaths: - cfgOut &= &"--nimblePath:\"{path}\"\n" + result &= &"--nimblePath:\"{path}\"\n" # Add `paths` in all cases if any detected for path in cfg.paths: - cfgOut &= &"--path:\"{path}\"\n" + result &= &"--path:\"{path}\"\n" when defined(windows): - cfgOut = cfgOut.replace("\\", "/") + result = result.replace("\\", "/") - writeFile(cfgFile, cfgOut) - -proc writeNimConfig*(cfgFile: string, projectDir = "") = +proc getNimConfigFlags*(projectDir = ""): string = + ## Get Nim command line configuration flags for `projectDir` + ## + ## If `projectDir` is not specified, it is detected if compile time or + ## current directory is used. let cfg = getNimConfig(projectDir) - writeNimConfig(cfg, cfgFile) + cfgOut = getNimConfigFlags(cfg) + return cfgOut.replace("\n", " ") + +proc writeNimConfig*(cfgFile: string, projectDir = "") = + ## Write Nim configuration for `projectDir` to specified `cfgFile` + ## + ## If `projectDir` is not specified, it is detected if compile time or + ## current directory is used. + let + cfg = getNimConfig(projectDir) + cfgOut = getNimConfigFlags(cfg) + writeFile(cfgFile, cfgOut) \ No newline at end of file From 2060c8964d6960808b86f767ea3949da9b3341e6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Apr 2020 11:54:00 -0500 Subject: [PATCH 088/255] Prevent plugin outdir override by configuration --- nimterop.nimble | 2 +- nimterop/getters.nim | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index a6fba1e..e520c71 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -10,7 +10,7 @@ installDirs = @["nimterop"] installFiles = @["config.nims"] # Dependencies -requires "nim >= 0.20.2", "regex#v0.13.1", "cligen >= 0.9.43" +requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" import nimterop/docs diff --git a/nimterop/getters.nim b/nimterop/getters.nim index d829cf0..ebd3432 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -731,7 +731,13 @@ proc loadPlugin*(gState: State, sourcePath: string) = flags = if fileExists(sourcePath & ".cfg"): "" else: getNimConfigFlags(getCurrentDir()) - cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {sourcePath.sanitizePath}" + + # Always set output to same directory as source, prevents override + outflags = &"--out:\"{pdll.extractFilename()}\" --outdir:\"{pdll.parentDir()}\"" + + # Compile plugin as library with `markAndSweep` GC + cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" + discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath From 76d9be756a540272cc7a910eb0730297643276f7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Apr 2020 12:52:08 -0500 Subject: [PATCH 089/255] Fix mode, toast output dir --- nimterop.nimble | 1 - nimterop/cimport.nim | 6 ++++-- nimterop/getters.nim | 2 +- config.nims => nimterop/toast.nims | 7 ++++++- tests/tsoloud.nims | 5 +++++ 5 files changed, 16 insertions(+), 5 deletions(-) rename config.nims => nimterop/toast.nims (75%) create mode 100644 tests/tsoloud.nims diff --git a/nimterop.nimble b/nimterop.nimble index e520c71..caaaf90 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -7,7 +7,6 @@ license = "MIT" bin = @["nimterop/toast"] installDirs = @["nimterop"] -installFiles = @["config.nims"] # Dependencies requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 4afb27c..47dc0eb 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -100,6 +100,8 @@ proc getToastError(output: string): string = # Filter out preprocessor errors for line in output.splitLines(): if "fatal error:" in line.toLowerAscii: + if result.len == 0: + result = "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" result &= "\n\nERROR:$1\n" % line.split("fatal error:")[1] # Toast error @@ -135,7 +137,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" let toastExe = toastExePath() doAssert fileExists(toastExe), "toast not compiled: " & toastExe.sanitizePath & " make sure 'nimble build' or 'nimble install' built it" - cmd &= &"{toastExe} --preprocess" + cmd &= &"{toastExe} --preprocess -m:{mode}" if recurse: cmd.add " --recurse" @@ -679,7 +681,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: echo "# Importing " & fullpath & " with c2nim" let - output = getToast(@[fullpath], recurse, dynlib, noNimout = true) + output = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) hash = output.hash().abs() hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" npath = hpath[0 .. hpath.rfind('.')] & "nim" diff --git a/nimterop/getters.nim b/nimterop/getters.nim index ebd3432..42a5a57 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -733,7 +733,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = else: getNimConfigFlags(getCurrentDir()) # Always set output to same directory as source, prevents override - outflags = &"--out:\"{pdll.extractFilename()}\" --outdir:\"{pdll.parentDir()}\"" + outflags = &"--out:\"{pdll}\"" # Compile plugin as library with `markAndSweep` GC cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" diff --git a/config.nims b/nimterop/toast.nims similarity index 75% rename from config.nims rename to nimterop/toast.nims index f5f7d21..94c9f20 100644 --- a/config.nims +++ b/nimterop/toast.nims @@ -1,3 +1,5 @@ +import os + # Workaround for C++ scanner.cc causing link error with other C obj files when defined(MacOSX): switch("clang.linkerexe", "g++") @@ -17,4 +19,7 @@ switch("path", "$nim") # Case objects when not defined(danger): - switch("define", "nimOldCaseObjects") \ No newline at end of file + switch("define", "nimOldCaseObjects") + +# Prevent outdir override +switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) \ No newline at end of file diff --git a/tests/tsoloud.nims b/tests/tsoloud.nims new file mode 100644 index 0000000..3b6174b --- /dev/null +++ b/tests/tsoloud.nims @@ -0,0 +1,5 @@ +# Workaround for C++ scanner.cc causing link error with other C obj files +when defined(MacOSX): + switch("clang.linkerexe", "g++") +else: + switch("gcc.linkerexe", "g++") \ No newline at end of file From 492eba8f22c6fb0070274590490f2977e357de2a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Apr 2020 20:48:21 -0500 Subject: [PATCH 090/255] Add timeit to measure test timings --- nimterop.nimble | 15 ++++++++++----- tests/getheader.nims | 7 +------ tests/timeit.nim | 22 ++++++++++++++++++++++ tests/wrappers.nims | 6 +++--- 4 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 tests/timeit.nim diff --git a/nimterop.nimble b/nimterop.nimble index caaaf90..8b6a756 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.4.4" +version = "0.5.0" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -14,8 +14,7 @@ requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" import nimterop/docs proc execCmd(cmd: string) = - echo "execCmd:" & cmd - exec cmd + exec "tests/timeit " & cmd proc execTest(test: string, flags = "") = execCmd "nim c --hints:off -f " & flags & " -r " & test @@ -24,6 +23,9 @@ proc execTest(test: string, flags = "") = task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") +task buildTimeit, "build timer": + exec "nim c -d:danger tests/timeit" + task bt, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") @@ -34,6 +36,7 @@ task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") task test, "Test": + buildTimeitTask() buildToastTask() execTest "tests/tast2.nim" @@ -66,8 +69,10 @@ task test, "Test": # getHeader tests withDir("tests"): - execCmd("nim e getheader.nims") + exec "nim e getheader.nims" if not existsEnv("APPVEYOR"): - execCmd("nim e wrappers.nims") + exec "nim e wrappers.nims" docsTask() + + echo readFile("tests/timeit.txt") \ No newline at end of file diff --git a/tests/getheader.nims b/tests/getheader.nims index ef501ab..72407f9 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -2,16 +2,11 @@ import strutils proc testCall(cmd, output: string, exitCode: int, delete = true) = var - ccmd = - when defined(windows): - "cmd /c " & cmd - else: - cmd + ccmd = "../tests/timeit " & cmd if not delete: ccmd = ccmd.replace(" -f ", " ") - echo ccmd var (outp, exitC) = gorgeEx(ccmd) echo outp diff --git a/tests/timeit.nim b/tests/timeit.nim new file mode 100644 index 0000000..b52eae5 --- /dev/null +++ b/tests/timeit.nim @@ -0,0 +1,22 @@ +import std/monotimes, os, osproc, sequtils, strformat, strutils, times + +when isMainModule: + var params = commandLineParams() + params.apply(quoteShell) + + let cmd = params.join(" ") + echo &"================\nRunning: {cmd}\n" + + let + + start = getMonoTime() + ret = execCmd(cmd) + endt = getMonoTime() + + outf = getAppDir() / "timeit.txt" + outd = if fileExists(outf): readFile(outf) else: "" + outp = &"\nRan: {cmd}\nTime taken: {$(endt - start)}\n" + + echo outp + writeFile(outf, outd & outp) + quit(ret) \ No newline at end of file diff --git a/tests/wrappers.nims b/tests/wrappers.nims index 37ac686..4687d7a 100644 --- a/tests/wrappers.nims +++ b/tests/wrappers.nims @@ -9,7 +9,7 @@ withDir("wrappers"): for wrapper in wrappers: let name = wrapper.extractFilename() - exec "git clone https://github.com/" & wrapper + exec "../../tests/timeit git clone https://github.com/" & wrapper withDir(name): - exec "nimble install -d" - exec "nimble test" \ No newline at end of file + exec "../../../tests/timeit nimble install -d" + exec "../../../tests/timeit nimble test" \ No newline at end of file From 9a93417ae21ddd11037505f58ad793aebea8644a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 29 Apr 2020 00:07:15 -0500 Subject: [PATCH 091/255] Fix timeit for legacy, reduce Windows test matrix --- nimterop.nimble | 6 ++++-- nimterop/build.nim | 18 +++++++++--------- nimterop/cimport.nim | 2 +- nimterop/docs.nim | 4 ++-- nimterop/nimconf.nim | 2 +- nimterop/template.nim | 2 +- tests/getheader.nims | 10 +++++----- tests/timeit.nim | 13 ++++++++++--- tests/tmath.nim | 2 +- tests/tnimterop_c.nim | 2 +- tests/tpcre.nim | 2 +- tests/zlib.nim | 2 +- 12 files changed, 37 insertions(+), 28 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8b6a756..579d315 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -70,8 +70,10 @@ task test, "Test": # getHeader tests withDir("tests"): exec "nim e getheader.nims" - if not existsEnv("APPVEYOR"): - exec "nim e wrappers.nims" + when not defined(Windows): + # Skip on Windows since very slow + if not existsEnv("APPVEYOR"): + exec "nim e wrappers.nims" docsTask() diff --git a/nimterop/build.nim b/nimterop/build.nim index e11cd7b..5bfec76 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -34,7 +34,7 @@ proc sleep*(milsecs: int) = ## Sleep at compile time let cmd = - when defined(windows): + when defined(Windows): "cmd /c timeout " else: "sleep " @@ -110,7 +110,7 @@ proc findExe*(exe: string): string = ## at compile time var cmd = - when defined(windows): + when defined(Windows): "where " & exe else: "which " & exe @@ -354,7 +354,7 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = ## `first`. Without it, the shortest match is returned. var cmd = - when defined(windows): + when defined(Windows): "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" elif defined(linux): "find $3 $1 -regextype egrep -regex $2" @@ -364,10 +364,10 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = recursive = "" if recurse: - when defined(windows): + when defined(Windows): recursive = "--recursive" else: - when not defined(windows): + when not defined(Windows): recursive = "-maxdepth 1" var @@ -388,7 +388,7 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = if ret == 0: for line in files.splitLines(): let f = - when defined(windows): + when defined(Windows): if ": " in line: line.split(": ", maxsplit = 1)[1] else: @@ -750,7 +750,7 @@ proc getLocalPath(header, outdir: string): string = result = findFile(header, outdir) proc getNumProcs(): string = - when defined(windows): + when defined(Windows): getEnv("NUMBER_OF_PROCESSORS").strip() elif defined(linux): execAction("nproc").output.strip() @@ -778,7 +778,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin if findExe("cmake").len != 0: var gen = "" - when defined(windows): + when defined(Windows): if findExe("sh").len != 0: let uname = execAction("sh -c uname -a").output.toLowerAscii() @@ -828,7 +828,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin result = findFile(lname, outdir, regex = true) proc getDynlibExt(): string = - when defined(windows): + when defined(Windows): result = ".dll" elif defined(linux) or defined(FreeBSD): result = ".so[0-9.]*" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 47dc0eb..4cb364d 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -623,7 +623,7 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## ## const ## dynpcre = - ## when defined(windows): + ## when defined(Windows): ## when defined(cpu64): ## "pcre64.dll" ## else: diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 6eb812b..d808535 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -19,7 +19,7 @@ proc getNimRootDir(): string = fmt"{currentSourcePath}".parentDir.parentDir.parentDir const - DirSep = when defined(windows): '\\' else: '/' + DirSep = when defined(Windows): '\\' else: '/' proc execAction(cmd: string): string = var @@ -53,7 +53,7 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath ## ## NOTE: `buildDocs()` only works correctly on Windows with Nim 1.0+ since ## https://github.com/nim-lang/Nim/pull/11814 is required. - when defined(windows) and (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + when defined(Windows) and (NimMajor, NimMinor, NimPatch) < (1, 0, 0): echo "buildDocs() unsupported on Windows for Nim < 1.0 - requires PR #11814" else: let diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 664675e..144a7e0 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -187,7 +187,7 @@ proc getNimConfigFlags(cfg: Config): string = for path in cfg.paths: result &= &"--path:\"{path}\"\n" - when defined(windows): + when defined(Windows): result = result.replace("\\", "/") proc getNimConfigFlags*(projectDir = ""): string = diff --git a/nimterop/template.nim b/nimterop/template.nim index f625280..c6e846f 100644 --- a/nimterop/template.nim +++ b/nimterop/template.nim @@ -70,7 +70,7 @@ cDefine("SYMBOL", "value") cCompile(srcDir/"file.c") # Perform OS specific tasks -when defined(windows): +when defined(Windows): # Windows specific symbols, options and files # Dynamic library to link against diff --git a/tests/getheader.nims b/tests/getheader.nims index 72407f9..2e6530f 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -35,6 +35,11 @@ when defined(posix): testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) + # dl - remove from Windows to save some time + testCall(cmd & " -d:lzmaDL" & lrcmd, "Need version", 1) + testCall(cmd & " -d:lzmaDL -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) + testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0, delete = false) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) @@ -43,11 +48,6 @@ testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) testCall(cmd & " -d:zlibGit -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0) testCall(cmd & " -d:zlibGit -d:zlibStatic -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0, delete = false) -# dl -testCall(cmd & " -d:lzmaDL" & lrcmd, "Need version", 1) -testCall(cmd & " -d:lzmaDL -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) -testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0, delete = false) - # dl testCall(cmd & " -d:zlibDL -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0) testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0, delete = false) diff --git a/tests/timeit.nim b/tests/timeit.nim index b52eae5..b448191 100644 --- a/tests/timeit.nim +++ b/tests/timeit.nim @@ -1,4 +1,11 @@ -import std/monotimes, os, osproc, sequtils, strformat, strutils, times +import os, osproc, sequtils, strformat, strutils, times + +when (NimMajor, NimMinor) >= (1, 0): + import std/monotimes + + template getTime(): MonoTime = getMonoTime() +else: + template getTime(): float = epochTime() when isMainModule: var params = commandLineParams() @@ -9,9 +16,9 @@ when isMainModule: let - start = getMonoTime() + start = getTime() ret = execCmd(cmd) - endt = getMonoTime() + endt = getTime() outf = getAppDir() / "timeit.txt" outd = if fileExists(outf): readFile(outf) else: "" diff --git a/tests/tmath.nim b/tests/tmath.nim index b8477c1..6c7999a 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -7,7 +7,7 @@ cOverride: mingw_ldbl_type_t = object mingw_dbl_type_t = object -when defined(windows): +when defined(Windows): cOverride: type complex = object diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index 0360225..8428489 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -45,7 +45,7 @@ check TEST_STR == "hello world" when defined(osx): check OSDEF == 10 -elif defined(windows): +elif defined(Windows): check OSDEF == 20 else: check OSDEF == 30 diff --git a/tests/tpcre.nim b/tests/tpcre.nim index c8e8059..4a426c4 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -14,7 +14,7 @@ static: const dynpcre = - when defined(windows): + when defined(Windows): when defined(cpu64): "pcre64.dll" else: diff --git a/tests/zlib.nim b/tests/zlib.nim index ce26c9a..09a6df2 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -15,7 +15,7 @@ proc zlibPreBuild(outdir, path: string) = # Delete default Makefile if mf.readFile().contains("configure first"): mf.rmFile() - when defined(windows): + when defined(Windows): # Fix static lib name on Windows setCmakeLibName(outdir, "zlibstatic", prefix = "lib", oname = "zlib", suffix = ".a") From 0098947a8161f4d832bffae34ac7a0b0a5341312 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 29 Apr 2020 13:02:06 -0500 Subject: [PATCH 092/255] Fix timeit ret for osx --- tests/timeit.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/timeit.nim b/tests/timeit.nim index b448191..cf9417f 100644 --- a/tests/timeit.nim +++ b/tests/timeit.nim @@ -26,4 +26,4 @@ when isMainModule: echo outp writeFile(outf, outd & outp) - quit(ret) \ No newline at end of file + quit(ret mod 255) From 2278f0e49d4da012370bb8e907640b9d89ae54fa Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 29 Apr 2020 14:38:35 -0500 Subject: [PATCH 093/255] Add -f:ast1, lesser debug output --- nimterop/globals.nim | 4 ++-- nimterop/toast.nim | 12 +++++++----- tests/lzma.nim | 1 - tests/tast2.nim | 1 - tests/tmath.nim | 1 - tests/tnimterop_c.nim | 1 - tests/tnimterop_cpp.nim | 1 - tests/tpcre.nim | 1 - tests/zlib.nim | 3 --- 9 files changed, 9 insertions(+), 16 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 5db17a3..b9352a4 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -106,7 +106,7 @@ type nodeBranch*: seq[string] Feature* = enum - ast2 + ast1, ast2 var gStateCT {.compiletime, used.} = new(State) @@ -123,7 +123,7 @@ when not declared(CIMPORT): # Redirect output to file when required template gecho*(args: string) = if gState.outputHandle.isNil: - echo args + stdout.writeLine(args) else: gState.outputHandle.writeLine(args) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 98045bf..7a41874 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -21,7 +21,7 @@ proc process(gState: State, path: string, astTable: AstTable) = elif gState.pnim: if Feature.ast2 in gState.feature: ast2.parseNim(gState, path, root) - else: + elif Feature.ast1 in gState.feature: ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code @@ -33,7 +33,7 @@ proc main( debug = false, defines: seq[string] = @[], dynlib: string = "", - feature: seq[Feature] = @[], + feature: seq[Feature] = @[Feature.ast1], includeHeader = false, includeDirs: seq[string] = @[], mode = "", @@ -118,13 +118,13 @@ proc main( # Process grammar into AST let astTable = - if Feature.ast2 notin gState.feature: + if Feature.ast1 in gState.feature: parseGrammar() else: nil if pgrammar: - if Feature.ast2 notin gState.feature: + if Feature.ast1 in gState.feature: # Print AST of grammar gState.printGrammar(astTable) elif source.nBl: @@ -137,8 +137,10 @@ proc main( if gState.pnim: if Feature.ast2 in gState.feature: ast2.printNim(gState) - else: + elif Feature.ast1 in gState.feature: ast.printNim(gState) + gecho """{.hint: "The legacy wrapper generation algorithm is deprecated and will be removed in the next release of Nimterop.".}""" + gecho """{.hint: "Refer to CHANGES.md for details on migrating to the new backend.".}""" # Close outputFile if outputFile.len != 0: diff --git a/tests/lzma.nim b/tests/lzma.nim index 1cc6aa0..f30b974 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -7,7 +7,6 @@ const flags = "--prefix=___,__,_ --suffix=__,_" static: - cDebug() cSkipSymbol(@[ "PRIX8", "PRIX16", "PRIX32", "PRIXLEAST8", "PRIXLEAST16", "PRIXLEAST32", diff --git a/tests/tast2.nim b/tests/tast2.nim index 4cfbeac..273d3e3 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -7,7 +7,6 @@ static: # the VM does not support it when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): cSkipSymbol @["CASTEXPR"] - cDebug() const path = currentSourcePath.parentDir() / "include" / "tast2.h" diff --git a/tests/tmath.nim b/tests/tmath.nim index 6c7999a..8f947b2 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -19,7 +19,6 @@ static: cSkipSymbol @["math_errhandling", "FP_ILOGB0", "FP_ILOGBNAN"] else: cSkipSymbol @["math_errhandling"] - cDebug() cDisableCaching() cAddStdDir() diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index 8428489..20c1678 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -3,7 +3,6 @@ import nimterop/cimport import nimterop/paths static: - cDebug() cDisableCaching() cAddSearchDir testsIncludeDir() diff --git a/tests/tnimterop_cpp.nim b/tests/tnimterop_cpp.nim index 14c8c8f..3f44e86 100644 --- a/tests/tnimterop_cpp.nim +++ b/tests/tnimterop_cpp.nim @@ -3,7 +3,6 @@ import nimterop/cimport import nimterop/paths static: - cDebug() cDisableCaching() cAddSearchDir testsIncludeDir() diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 4a426c4..51530c1 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -9,7 +9,6 @@ const static: if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) - cDebug() cDisableCaching() const diff --git a/tests/zlib.nim b/tests/zlib.nim index 09a6df2..febc6b1 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -5,9 +5,6 @@ import nimterop/[build, cimport] const baseDir = getProjectCacheDir("nimterop" / "tests" / "zlib") -static: - cDebug() - proc zlibPreBuild(outdir, path: string) = let mf = outdir / "Makefile" From 4784616d987c3b7c73125af44bec999184444a22 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 30 Apr 2020 08:54:18 -0500 Subject: [PATCH 094/255] Clean globals, build debug --- nimterop.nimble | 2 +- nimterop/ast2.nim | 2 +- nimterop/build.nim | 24 ++++++++++++++++------ nimterop/cimport.nim | 7 +++---- nimterop/globals.nim | 47 +++++++++++++++++++++----------------------- nimterop/paths.nim | 1 - nimterop/toast.nim | 5 ++++- nimterop/toast.nims | 5 ++++- tests/getheader.nims | 2 +- tests/tsoloud.nim | 1 - 10 files changed, 54 insertions(+), 42 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 579d315..a6ece7c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -24,7 +24,7 @@ task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") task buildTimeit, "build timer": - exec "nim c -d:danger tests/timeit" + exec "nim c --hints:off -d:danger tests/timeit" task bt, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 474ec69..13f2ba2 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -6,7 +6,7 @@ import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser, comphelp, tshelp] +import "."/[comphelp, exprparser, globals, getters, tshelp] proc getPtrType*(str: string): string = result = case str: diff --git a/nimterop/build.nim b/nimterop/build.nim index 5bfec76..6fa44cb 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,6 +4,18 @@ import os except findExe, sleep import regex +# build specific debug since we cannot import globals (yet) +var + gDebug* = false + gDebugCT* {.compileTime.} = false + +proc echoDebug(str: string) = + let str = "\n# " & str.strip().replace("\n", "\n# ") + when nimvm: + if gDebugCT: echo str + else: + if gDebug: echo str + proc fixCmd(cmd: string): string = when defined(Windows): # Replace 'cd d:\abc' with 'd: && cd d:\abc` @@ -461,7 +473,7 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / i): echo "# Running autogen.sh" - echo execAction( + echoDebug execAction( &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output break @@ -471,7 +483,7 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / i): echo "# Running autoreconf" - echo execAction(&"cd {path.sanitizePath} && autoreconf -fi").output + echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output break @@ -483,7 +495,7 @@ proc configure*(path, check: string, flags = "") = if flags.len != 0: cmd &= &" {flags}" - echo execAction(cmd).output + echoDebug execAction(cmd).output doAssert (path / check).fileExists(), "# Configure failed" @@ -577,10 +589,10 @@ proc cmake*(path, check, flags: string) = mkDir(path) - var + let cmd = &"cd {path.sanitizePath} && cmake {flags}" - echo execAction(cmd).output + echoDebug execAction(cmd).output doAssert (path / check).fileExists(), "# cmake failed" @@ -616,7 +628,7 @@ proc make*(path, check: string, flags = "", regex = false) = if flags.len != 0: cmd &= &" {flags}" - echo execAction(cmd).output + echoDebug execAction(cmd).output doAssert findFile(check, path, regex = regex).len != 0, "# make failed" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 4cb364d..e081633 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,11 +17,9 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -const CIMPORT {.used.} = 1 +import regex -include "."/globals - -import "."/[build, paths, types] +import "."/[build, globals, paths, types] export types proc interpPath(dir: string): string= @@ -393,6 +391,7 @@ proc cSearchPath*(path: string): string {.compileTime.}= proc cDebug*() {.compileTime.} = ## Enable debug messages and display the generated Nim code gStateCT.debug = true + build.gDebugCT = true proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development diff --git a/nimterop/globals.nim b/nimterop/globals.nim index b9352a4..ae9a6d0 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -4,13 +4,13 @@ import regex import "."/plugin -when not declared(CIMPORT): +when defined(TOAST): import compiler/[ast, idents, modulegraphs, options] import "."/treesitter/api const - gAtoms {.used.} = @[ + gAtoms* {.used.} = @[ "field_identifier", "identifier", "number_literal", @@ -21,7 +21,7 @@ const "type_identifier" ].toHashSet() - gExpressions {.used.} = @[ + gExpressions* {.used.} = @[ "parenthesized_expression", "bitwise_expression", "shift_expression", @@ -29,32 +29,32 @@ const "escape_sequence" ].toHashSet() - gEnumVals {.used.} = @[ + gEnumVals* {.used.} = @[ "identifier", "number_literal", "char_literal" ].concat(toSeq(gExpressions.items)) type - Kind = enum + Kind* = enum exactlyOne oneOrMore # + zeroOrMore # * zeroOrOne # ? orWithNext # ! - Ast = object + Ast* = object name*: string kind*: Kind recursive*: bool children*: seq[ref Ast] - when not declared(CIMPORT): + when defined(TOAST): tonim*: proc (ast: ref Ast, node: TSNode, gState: State) regex*: Regex - AstTable {.used.} = TableRef[string, seq[ref Ast]] + AstTable* {.used.} = TableRef[string, seq[ref Ast]] - State = ref object + State* = ref object compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string] debug*, includeHeader*, nocache*, nocomments*, past*, preprocess*, pnim*, recurse*: bool @@ -87,7 +87,7 @@ type commentStr*, debugStr*, skipStr*: string # Nim compiler objects - when not declared(CIMPORT): + when defined(TOAST): constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode identCache*: IdentCache config*: ConfigRef @@ -109,24 +109,21 @@ type ast1, ast2 var - gStateCT {.compiletime, used.} = new(State) + gStateCT* {.compiletime, used.} = new(State) -template nBl(s: typed): untyped {.used.} = +template nBl*(s: typed): untyped {.used.} = (s.len != 0) -template Bl(s: typed): untyped {.used.} = +template Bl*(s: typed): untyped {.used.} = (s.len == 0) -when not declared(CIMPORT): - export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl +# Redirect output to file when required +template gecho*(args: string) = + if gState.outputHandle.isNil: + stdout.writeLine(args) + else: + gState.outputHandle.writeLine(args) - # Redirect output to file when required - template gecho*(args: string) = - if gState.outputHandle.isNil: - stdout.writeLine(args) - else: - gState.outputHandle.writeLine(args) - - template decho*(args: varargs[string, `$`]): untyped = - if gState.debug: - gecho join(args, "").getCommented() \ No newline at end of file +template decho*(args: varargs[string, `$`]): untyped = + if gState.debug: + gecho join(args, "").getCommented() \ No newline at end of file diff --git a/nimterop/paths.nim b/nimterop/paths.nim index 0ae461f..fa245b3 100644 --- a/nimterop/paths.nim +++ b/nimterop/paths.nim @@ -16,4 +16,3 @@ proc toastExePath*(): string = proc testsIncludeDir*(): string = nimteropRoot() / "tests" / "include" - diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 7a41874..78708f5 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,7 @@ import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, globals, getters, grammar, build, tshelp] +import "."/[ast, ast2, build, globals, getters, grammar, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" @@ -81,6 +81,9 @@ proc main( doAssert not (includeHeader == true and dynlib.nBl), "`includeHeader` and `dynlib` cannot be used simultaneously" + # Set gDebug in build.nim + build.gDebug = debug + # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/nimterop/toast.nims b/nimterop/toast.nims index 94c9f20..83d5cf4 100644 --- a/nimterop/toast.nims +++ b/nimterop/toast.nims @@ -22,4 +22,7 @@ when not defined(danger): switch("define", "nimOldCaseObjects") # Prevent outdir override -switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) \ No newline at end of file +switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) + +# Define TOAST for globals.nim +switch("define", "TOAST") \ No newline at end of file diff --git a/tests/getheader.nims b/tests/getheader.nims index 2e6530f..1e8027a 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -14,7 +14,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = doAssert outp.contains(output), outp var - cmd = "nim c -f" + cmd = "nim c -f --hints:off" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" lexp = "liblzma version = " diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index afff0f1..8dd197c 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -7,7 +7,6 @@ const static: gitPull("https://github.com/jarikomppa/soloud", baseDir, "include/*\nsrc/*\n", checkout = "RELEASE_20200207") - cDebug() cDisableCaching() cOverride: From 34043bd5af52d20df1e85f28c4369d5810248017 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 1 May 2020 07:20:45 -0600 Subject: [PATCH 095/255] Fix #198: integer out of range --- nimterop/exprparser.nim | 44 +++++++++++++++++++++++++++-------------- tests/include/tast2.h | 3 +++ tests/tast2.nim | 2 ++ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index c74f0b6..e68c0b1 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -133,13 +133,38 @@ proc getFloatNode(number, suffix: string): PNode {.inline.} = proc getIntNode(number, suffix: string): PNode {.inline.} = ## Get a Nim int node from a C integer expression + suffix + var + val: BiggestInt + flags: TNodeFlags + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + val = parseHexInt(number) + flags = {nfBase16} + elif number.contains(re"0[bB]"): + val = parseBinInt(number) + flags = {nfBase2} + elif number.contains(re"0[oO]"): + val = parseOctInt(number) + flags = {nfBase8} + else: + val = parseInt(number) + case suffix of "u", "U": result = newNode(nkUintLit) of "l", "L": - result = newNode(nkInt32Lit) + # If the value doesn't fit, adjust + if val > int32.high or val < int32.low: + result = newNode(nkInt64Lit) + else: + result = newNode(nkInt32Lit) of "ul", "UL": - result = newNode(nkUint32Lit) + # If the value doesn't fit, adjust + if val > uint32.high.BiggestInt: + result = newNode(nkUInt64Lit) + else: + result = newNode(nkUInt32Lit) of "ll", "LL": result = newNode(nkInt64Lit) of "ull", "ULL": @@ -147,19 +172,8 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = else: result = newNode(nkIntLit) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) + result.intVal = val + result.flags = flags proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode diff --git a/tests/include/tast2.h b/tests/include/tast2.h index b47a801..e1d4529 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -33,6 +33,9 @@ extern "C" { #define EQ5 AVAL != BVAL #define EQ6 AVAL == BVAL +// testing integer out of long int range +#define INT_FAST16_MIN (-9223372036854775807L-1) + #define SIZEOF sizeof(char) #define REG_STR "regular string" #define NOTSUPPORTEDSTR "not a " REG_STR diff --git a/tests/tast2.nim b/tests/tast2.nim index 273d3e3..54b89e5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -131,6 +131,8 @@ assert SIZEOF == 1 assert COERCE == 645635670332'u64 assert COERCE2 == 645635670332'i64 +assert INT_FAST16_MIN == -9223372036854775807'i64 - 1 + assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 From 08f8ca32f41d16ade0b364e048a0a99b1a01eebe Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 1 May 2020 11:02:29 -0500 Subject: [PATCH 096/255] Fix #196 - more known types --- nimterop/ast2.nim | 4 +--- nimterop/getters.nim | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 13f2ba2..3ffa3f6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -42,8 +42,6 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS else: gecho &"\n# $1'{origname}' skipped" % skind gState.skippedSyms.incl origname - if gState.debug: - gState.skipStr &= &"\n{gState.getNodeVal(node)}" proc addOverrideFinal(gState: State, kind: NimSymKind) = # Add all unused cOverride symbols for `kind` to AST @@ -285,7 +283,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: if name.Bl: # Name skipped or overridden since blank result = gState.getOverrideOrSkip(node, origname, kind) - elif gState.addNewIdentifer(name): + elif origname notin gTypeMap and gState.addNewIdentifer(name): if kind == nskType: # type name* = # diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 42a5a57..69628f5 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -57,6 +57,30 @@ const gTypeMap* = { "u_int": "cuint", "size_t": "uint", + "int8_t": "int8", + "int16_t": "int16", + "int32_t": "int32", + "int64_t": "int64", + + "intptr_t": "ptr int", + + "Int8": "int8", + "Int16": "int16", + "Int32": "int32", + "Int64": "int64", + + "uint8_t": "uint8", + "uint16_t": "uint16", + "uint32_t": "uint32", + "uint64_t": "uint64", + + "uintptr_t": "ptr uint", + + "Uint8": "uint8", + "Uint16": "uint16", + "Uint32": "uint32", + "Uint64": "uint64", + # long "long": "clong", "long int": "clong", @@ -87,10 +111,7 @@ proc getType*(str: string): string = if str == "void": return "object" - result = str.strip(chars={'_'}). - replace(re"\s+", " "). - replace(re"^([u]?int[\d]+)_t$", "$1"). - replace(re"^([u]?int)ptr_t$", "ptr $1") + result = str.strip(chars={'_'}).replace(re"\s+", " ") if gTypeMap.hasKey(result): result = gTypeMap[result] From 748998b0bd921a60f1c05aae7b47eb8de51b7e3e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 1 May 2020 15:07:51 -0500 Subject: [PATCH 097/255] Fixes and test lzma/zlib with ast2 --- nimterop/ast2.nim | 6 +- nimterop/getters.nim | 134 ++++++++++++++++++++++--------------------- tests/getheader.nims | 2 +- tests/lzma.nim | 8 ++- tests/tsoloud.nim | 3 +- tests/zlib.nim | 6 +- 6 files changed, 85 insertions(+), 74 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 3ffa3f6..4d609cb 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -283,7 +283,9 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: if name.Bl: # Name skipped or overridden since blank result = gState.getOverrideOrSkip(node, origname, kind) - elif origname notin gTypeMap and gState.addNewIdentifer(name): + elif name notin gTypeMapValues and gState.addNewIdentifer(name): + # Add only if not an existing Nim type + if kind == nskType: # type name* = # @@ -1399,7 +1401,7 @@ proc addEnum(gState: State, node: TSNode) = continue let fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField) - if fname.nBl: + if fname.nBl and gState.addNewIdentifer(fname): var fval = "" if prev.Bl: diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 69628f5..d94fae8 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -30,82 +30,86 @@ yield""".split(Whitespace).toHashSet() # Types related -const gTypeMap* = { - # char - "char": "cchar", - "signed char": "cschar", - "unsigned char": "cuchar", +const + gTypeMap* = { + # char + "char": "cchar", + "signed char": "cschar", + "unsigned char": "cuchar", - # short - "short": "cshort", - "short int": "cshort", - "signed short": "cshort", - "signed short int": "cshort", - "unsigned short": "cushort", - "unsigned short int": "cushort", - "uShort": "cushort", - "u_short": "cushort", + # short + "short": "cshort", + "short int": "cshort", + "signed short": "cshort", + "signed short int": "cshort", + "unsigned short": "cushort", + "unsigned short int": "cushort", + "uShort": "cushort", + "u_short": "cushort", - # int - "int": "cint", - "signed": "cint", - "signed int": "cint", - "ssize_t": "int", - "unsigned": "cuint", - "unsigned int": "cuint", - "uInt": "cuint", - "u_int": "cuint", - "size_t": "uint", + # int + "int": "cint", + "signed": "cint", + "signed int": "cint", + "ssize_t": "int", + "unsigned": "cuint", + "unsigned int": "cuint", + "uInt": "cuint", + "u_int": "cuint", + "size_t": "uint", - "int8_t": "int8", - "int16_t": "int16", - "int32_t": "int32", - "int64_t": "int64", + "int8_t": "int8", + "int16_t": "int16", + "int32_t": "int32", + "int64_t": "int64", - "intptr_t": "ptr int", + "intptr_t": "ptr int", - "Int8": "int8", - "Int16": "int16", - "Int32": "int32", - "Int64": "int64", + "Int8": "int8", + "Int16": "int16", + "Int32": "int32", + "Int64": "int64", - "uint8_t": "uint8", - "uint16_t": "uint16", - "uint32_t": "uint32", - "uint64_t": "uint64", + "uint8_t": "uint8", + "uint16_t": "uint16", + "uint32_t": "uint32", + "uint64_t": "uint64", - "uintptr_t": "ptr uint", + "uintptr_t": "ptr uint", - "Uint8": "uint8", - "Uint16": "uint16", - "Uint32": "uint32", - "Uint64": "uint64", + "Uint8": "uint8", + "Uint16": "uint16", + "Uint32": "uint32", + "Uint64": "uint64", - # long - "long": "clong", - "long int": "clong", - "signed long": "clong", - "signed long int": "clong", - "off_t": "clong", - "unsigned long": "culong", - "unsigned long int": "culong", - "uLong": "culong", - "u_long": "culong", + # long + "long": "clong", + "long int": "clong", + "signed long": "clong", + "signed long int": "clong", + "off_t": "clong", + "unsigned long": "culong", + "unsigned long int": "culong", + "uLong": "culong", + "u_long": "culong", - # long long - "long long": "clonglong", - "long long int": "clonglong", - "signed long long": "clonglong", - "signed long long int": "clonglong", - "off64_t": "clonglong", - "unsigned long long": "culonglong", - "unsigned long long int": "culonglong", + # long long + "long long": "clonglong", + "long long int": "clonglong", + "signed long long": "clonglong", + "signed long long int": "clonglong", + "off64_t": "clonglong", + "unsigned long long": "culonglong", + "unsigned long long int": "culonglong", - # floating point - "float": "cfloat", - "double": "cdouble", - "long double": "clongdouble" -}.toTable() + # floating point + "float": "cfloat", + "double": "cdouble", + "long double": "clongdouble" + }.toTable() + + # Nim type names that shouldn't need to be wrapped again + gTypeMapValues* = toSeq(gTypeMap.values).toHashSet() proc getType*(str: string): string = if str == "void": diff --git a/tests/getheader.nims b/tests/getheader.nims index 1e8027a..987545a 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -14,7 +14,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = doAssert outp.contains(output), outp var - cmd = "nim c -f --hints:off" + cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\"" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" lexp = "liblzma version = " diff --git a/tests/lzma.nim b/tests/lzma.nim index f30b974..cff39de 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -3,8 +3,10 @@ import os, strutils import nimterop/[build, cimport] const + FLAGS {.strdefine.} = "" + baseDir = getProjectCacheDir("nimterop" / "tests" / "liblzma") - flags = "--prefix=___,__,_ --suffix=__,_" + tflags = "--prefix=___,__,_ --suffix=__,_ " & FLAGS static: cSkipSymbol(@[ @@ -38,8 +40,8 @@ cOverride: lzma_index_iter = object when not lzmaStatic: - cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath", flags = flags) + cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath", flags = tflags) else: - cImport(lzmaPath, recurse = true, flags = flags) + cImport(lzmaPath, recurse = true, flags = tflags) echo "liblzma version = " & $lzma_version_string() diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index 8dd197c..4d9a28b 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -1,6 +1,8 @@ import os, nimterop/[cimport, build] const + FLAGS {.strdefine.} = "" + baseDir = getProjectCacheDir("nimterop" / "tests" / "soloud") incl = baseDir/"include" src = baseDir/"src" @@ -42,7 +44,6 @@ cCompile(src/"audiosource", "cpp", exclude="ay/") cCompile(src/"audiosource", "c") cCompile(src/"filter/*.cpp") -const FLAGS {.strdefine.} = "" cImport(incl/"soloud_c.h", flags = FLAGS) var diff --git a/tests/zlib.nim b/tests/zlib.nim index febc6b1..852dca9 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -3,6 +3,8 @@ import os, strutils import nimterop/[build, cimport] const + FLAGS {.strdefine.} = "" + baseDir = getProjectCacheDir("nimterop" / "tests" / "zlib") proc zlibPreBuild(outdir, path: string) = @@ -64,8 +66,8 @@ when zlibGit or zlibDL: cIncludeDir(baseDir / "buildcache") when not zlibStatic: - cImport(zlibPath, recurse = true, dynlib = "zlibLPath") + cImport(zlibPath, recurse = true, dynlib = "zlibLPath", flags = FLAGS) else: - cImport(zlibPath, recurse = true) + cImport(zlibPath, recurse = true, flags = FLAGS) echo "zlib version = " & $zlibVersion() From 370f64d4e799f5441ffc7b5cb0f3b012c09a13db Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 1 May 2020 17:51:57 -0500 Subject: [PATCH 098/255] Fix for zlib --- nimterop/getters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index d94fae8..620c1ec 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -565,7 +565,7 @@ proc getPreprocessor*(gState: State, fullpath: string): string = else: cmd &= "-D__attribute__(x)= " - cmd &= "-D__restrict= " + cmd &= "-D__restrict= -D__extension__= " cmd &= &"{fullpath.sanitizePath}" From ad240abdecd50ee3f8836624f403aaa0f0ded230 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 2 May 2020 23:48:20 -0500 Subject: [PATCH 099/255] Update README based on feedback --- README.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 53b37e7..6fbc040 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ [![Build status](https://ci.appveyor.com/api/projects/status/hol1yvqbp6hq4ao8/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimterop-8jcj7/branch/master) [![Build Status](https://travis-ci.org/nimterop/nimterop.svg?branch=master)](https://travis-ci.org/nimterop/nimterop) -Detailed documentation [here](https://nimterop.github.io/nimterop/theindex.html). - Nimterop is a [Nim](https://nim-lang.org/) package that aims to make C/C++ interop seamless -Most of the wrapping functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how `c2nim` can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application and other nimgen functionality that helps in automating the wrapping process. There is also support to statically or dynamically link to system installed libraries or downloading and building them with `autoconf` or `cmake` from a Git repo or source archive. +Most of the wrapping functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how `c2nim` can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application and other functionality that helps in automating the wrapping process. There is also support to statically or dynamically link to system installed libraries or downloading and building them with `autoconf` or `cmake` from a Git repo or source archive. The nimterop wrapping functionality is still limited to C but is constantly expanding. C++ support will be added once most popular C libraries can be wrapped seamlessly. Meanwhile, `c2nim` can also be used in place of `toast` with the `c2nImport()` API call. -Nimterop has seen some adoption within the community and the simplicity and success of this approach justifies additional investment of time and effort. Regardless, the goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation. +The goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation. ## Installation @@ -23,18 +21,22 @@ or: ```bash git clone http://github.com/nimterop/nimterop && cd nimterop nimble develop -y -nimble build +nimble build -d:danger ``` This will download and install nimterop in the standard Nimble package location, typically `~/.nimble`. Once installed, it can be imported into any Nim program. ## Usage -Detailed documentation can be found [here](https://nimterop.github.io/nimterop/theindex.html). Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work. +Nimterop can be used in two ways: +- Creating a wrapper file - a `.nim` file that contains calls to the high-level API that can download and build the C library as well as generate the required Nim code to interface with the library. This wrapper file can then be imported into Nim code like any other module and it will be processed at compile time. +- Using the command line `toast` tool to generate the Nim code which can then be stored into a file and imported separately. + +Any combination of the above is possible - only download, build or wrapping and nimterop avoids imposing any particular workflow. ### Build API -Creating a wrapper has two parts, the first is to setup the C library. This includes downloading it or finding it if already installed, and building it if applicable. The `getHeader()` high-level API provides all of this functionality as a convenience. Following is an example of using the high-level `getHeader()` API to perform all building and linking automatically: +Creating a wrapper has two parts, the first is to setup the C library. This includes downloading it or finding it if already installed, and building it if applicable. The `getHeader()` high-level API provides all of this functionality as a convenience. The following `.nim` wrapper file is an example of using the high-level `getHeader()` API to perform all building, wrapping and linking automatically: ```nim import nimterop/[build, cimport] @@ -63,9 +65,11 @@ else: cImport(headerPath, recurse = true) ``` +Module documentation for the build API can be found [here](https://nimterop.github.io/nimterop/build.html). Refer to the ```tests``` directory for additional examples on how the library can be used. Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work. + __Download / Search__ -The above wrapper is generic and allows the end user to control how it works. +The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. - If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. @@ -105,7 +109,7 @@ If more fine-tuned control is desired over the build process, it is possible to ### Wrapper API -Once the C library is setup, the next step is to create wrappers that inform Nim of all the types and functions that are available. Following is a simple example covering the API: +Once the C library is setup, the next step is to generate code that inform Nim of all the types and functions that are available. Following is a simple example covering the API: ```nim import nimterop/cimport @@ -124,7 +128,7 @@ cImport("clib.h") # Generate wrappers for header specified cCompile("clib/src/*.c") # Compile in any implementation source files ``` -Refer to the ```tests``` directory for additional examples on how the library can be used. The [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) is also a good source of examples. +Module documentation for the wrapper API can be found [here](https://nimterop.github.io/nimterop/cimport.html). __Preprocessing__ @@ -152,6 +156,10 @@ __Compiling source__ The job of building and compiling the underlying C library is best left to the build mechanism selected by the library author so using `getHeader()` is recommended. For simpler projects with a few `.c` files though, `cCompile()` should be more than enough. It is not recommended for larger projects which heavily rely on functionality offered by build tools. Recreating reliable logic in Nim can be tedious and one can expect minimal support from that author if their tested build mechanism is not used. +### Docs API + +Nimterop also provides a [docs](https://nimterop.github.io/nimterop/docs.html) API which can be used to generate documentation from the generated wrappers. This can be added as a task in the `.nimble` or `.nims` file for convenience. See [nimarchive.nimble](https://github.com/genotrance/nimarchive/blob/master/nimarchive.nimble) for an example. + ### Command line API The `toast` binary can also be used directly on the CLI, similar to `c2nim`. The `cPlugin()` interface From 35b0b5eff7ea789afd4bde56344ffdaa75ce31b3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 4 May 2020 12:01:05 -0500 Subject: [PATCH 100/255] Add change log --- CHANGES.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..b02453c --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,80 @@ +# Nimterop Change History + +## Version 0.5.0 + +This release introduces a new backend for wrapper generation dubbed `ast2` that leverages the Nim compiler AST and renderer. The new design simplifies feature development and already includes all the functionality of the legacy algorithm plus fixes for several open issues. + +The new backend can be leveraged with the `-f:ast2` flag to `toast` or `flags = "-f:ast2"` to `cImport()`. The legacy algorithm will be the default backend for this release but no new functionality or bugfixes are expected going forward. Usage of the legacy algorithm will display a *deprecated* hint to encourage users to test their wrappers with `-f:ast2` and remove any overrides that the new algorithm supports. + +Version 0.6.0 of Nimterop will make `ast2` the default backend and the legacy algorithm will be removed altogether. + +See the full list of changes here: + +https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 + +### Breaking changes + +- Nimterop now skips generating the `{.header.}` pragma by default in non-dynlib mode. This skips the header file `#include` in the generated code and allows creation of wrappers that do not require presence of the header during compile time. There are cases where this will not work so the `--includeHeader | -H` flag is available to revert to the legacy behavior when required. This change applies to both `ast2` and the legacy backend so if an existing wrapper breaks, test it with `-H` to see if it starts working again. [#169][i169] + +- Nimterop defaulted to C++ mode for preprocessing and tree-sitter parsing in all cases unless explicitly informed to use C mode. This has been changed and is now detected based on the file extension. This means some existing wrappers could break since they might contain C++ code or include C++ headers like `#include ` which will not work in C mode. Explicitly setting `mode = "cpp"` or `-mcpp` should fix such issues. [#176][i176] + +- Enums were originally being mapped to `distint int` - this has been changed to `distinct cint` since the sizes are incorrect on 64-bit and is especially noticeable when types or unions have enum fields. + +- `static inline` functions are no longer wrapped by the legacy backend. The `ast2` backend correctly generates wrappers for such functions but they are only generated when `--includeHeader | -H` is in effect. This is because such functions do not exist in the binary and can only be referenced when the header is compiled in. + +- Support for Nim v0.19.6 has been dropped and the test matrix now covers v0.20.2, v1.0.6, v1.2.0 and devel. + +### New functionality + +- `ast2` includes support for various C constructs that were issues with the legacy backend. These changes should reduce the reliance on `cOverride()` and existing wrappers should attempt to clean up such sections where possible. + - N-dimensional arrays and pointers - [#54][i54] + - Synomyms for types - [#74][i74] + - Varargs support - [#76][i76] + - Nested structs, unions and enums - [#137][i137] [#147][i147] + - Forward declarations of types - [#148][i148] + - Nested function pointers - [#155][i155] [#156][i156] + - Various enum fixes - [#159][i159] [#171][i171] + - Map `int arr[]` to `arr: UncheckedArray[cint]` - [#174][i174] + +- `ast2` also includes an advanced expression parser that can reliably handle constructs typically seen with `#define` statements and enumeration values: + - Integers + integer like expressions (hex, octal, suffixes) + - Floating point expressions + - Strings and character literals, including C's escape characters + - Math operators `+ - / *` + - Some Unary operators `- ! ~` + - Any identifiers + - C type descriptors `int char` etc + - Boolean values `true false` + - Shift, cast, math or sizeof expressions + - Most type coercions + +- Wrappers can now point to an external plugin file with `cPluginPath()` instead of having to declaring plugins inline with `cPlugin()`. This allows multiple wrappers to share the same plugin. [#181][i181] + +- `cImport()` adds support for importing multiple headers in a single call - this enables support for libraries that have many header files that include shared headers and typically cannot be imported in multiple `cImport()` calls since it results in duplicate symbols. Calling `toast` with multiple headers uses the same algorithm. + +- `ast2` now creates Nim doc comments instead of reqular comments which get rendered when the wrapper is run through `nim doc` or the `buildDocs()` API. [#197][i197] + +- `toast` now includes `--replace | -G` to manipulate identifier names beyond `--prefix` and `--suffix`. `-G:X=Y` replaces X with Y and `-G:@_[_]+=_` replaces multiple `_` with a single instance using the `@` prefix to enable regular expressions. + +- Nimterop is now able to detect Nim configuration of projects and can better handle cases where defaults such as `nimcacheDir` or `nimblePath` are overridden. This especially enables better interop with workflows that do not depend on Nimble. [#151][i151] [#153][i153] + +- Nimterop defaults to `cmake`, followed by `autoconf` for building libraries with `getHeader()`. It is now possible to change the order of discovery with the `buildType` value. [#200][i200] + +[i54]: https://github.com/nimterop/nimterop/issues/54 +[i74]: https://github.com/nimterop/nimterop/issues/74 +[i76]: https://github.com/nimterop/nimterop/issues/76 +[i137]: https://github.com/nimterop/nimterop/issues/137 +[i147]: https://github.com/nimterop/nimterop/issues/147 +[i148]: https://github.com/nimterop/nimterop/issues/148 +[i151]: https://github.com/nimterop/nimterop/issues/151 +[i153]: https://github.com/nimterop/nimterop/issues/153 +[i155]: https://github.com/nimterop/nimterop/issues/155 +[i156]: https://github.com/nimterop/nimterop/issues/156 +[i159]: https://github.com/nimterop/nimterop/issues/159 +[i169]: https://github.com/nimterop/nimterop/issues/169 +[i171]: https://github.com/nimterop/nimterop/issues/171 +[i174]: https://github.com/nimterop/nimterop/issues/174 +[i176]: https://github.com/nimterop/nimterop/issues/176 +[i181]: https://github.com/nimterop/nimterop/issues/181 +[i197]: https://github.com/nimterop/nimterop/issues/197 +[i200]: https://github.com/nimterop/nimterop/issues/200 \ No newline at end of file From 6cfe59debce37dccf1a118f83f3b15050bcbab09 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 4 May 2020 15:48:34 -0500 Subject: [PATCH 101/255] Update README with changelog, credits --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6fbc040..8680048 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ Nimterop can be used in two ways: Any combination of the above is possible - only download, build or wrapping and nimterop avoids imposing any particular workflow. +Refer to [CHANGES.md](CHANGES.md) for history and information around breaking changes. + ### Build API Creating a wrapper has two parts, the first is to setup the C library. This includes downloading it or finding it if already installed, and building it if applicable. The `getHeader()` high-level API provides all of this functionality as a convenience. The following `.nim` wrapper file is an example of using the high-level `getHeader()` API to perform all building, wrapping and linking automatically: @@ -224,6 +226,8 @@ This is part of the reason why Nimterop provides a wrapper API so that the gener Nimterop depends on [tree-sitter](http://tree-sitter.github.io/tree-sitter/) and all licensing terms of [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/master/LICENSE) apply to the usage of this package. The tree-sitter functionality is pulled and wrapped using nimterop itself. +Thank you to all the [contributors](https://github.com/nimterop/nimterop/graphs/contributors), issue submitters, various people in [#nim](irc://freenode.net/nim) and users for helping improve Nimterop over the years. + ## Feedback Nimterop is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/nimterop/nimterop) with an MIT license so issues, forks and PRs are most appreciated. From 0bf2021fd47f0c5ed53219dccfff1ad0d3a13650 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 2 May 2020 12:36:55 -0600 Subject: [PATCH 102/255] Make getHeader specify which make strategy to use Fix make call Add comment --- nimterop/build.nim | 103 +++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 6fa44cb..fe8f31a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,6 +4,15 @@ import os except findExe, sleep import regex +type + MakeType* = enum + mtConfMake, mtCMake + + BuildStatus = object + built: bool + buildPath: string + error: string + # build specific debug since we cannot import globals (yet) var gDebug* = false @@ -771,20 +780,7 @@ proc getNumProcs(): string = else: "1" -proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): string = - var - conDeps = false - conDepStr = "" - cmakeDeps = false - cmakeDepStr = "" - lpath = findFile(lname, outdir, regex = true) - makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" - made = false - makePath = outdir - - if lpath.len != 0: - return lpath - +proc buildWithCmake(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if fileExists(outdir / "CMakeLists.txt"): if findExe("cmake").len != 0: @@ -806,36 +802,59 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin gen = "Unix Makefiles".quoteShell if findExe("ccache").len != 0: gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - makePath = outdir / "buildcache" - cmake(makePath, "Makefile", &".. -G {gen} {cmakeFlags}") - cmakeDeps = true + result.buildPath = outdir / "buildcache" + cmake(result.buildPath, "Makefile", &".. -G {gen} {flags}") + result.built = true else: - cmakeDepStr &= "cmake executable missing" + result.error = "cmake capable but cmake executable missing" + else: + result.buildPath = outdir - if not cmakeDeps: - if findExe("bash").len != 0: - for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: - if fileExists(outdir / file): - configure(outdir, "Makefile", conFlags) - conDeps = true - - break - else: - conDepStr &= "bash executable missing" - - if fileExists(makePath / "Makefile"): - make(makePath, lname, makeFlagsProc, regex = true) - made = true +proc buildWithConfMake(outdir, flags: string): BuildStatus = + if not fileExists(outdir / "Makefile"): + if findExe("bash").len != 0: + for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: + if fileExists(outdir / file): + configure(outdir, "Makefile", flags) + result.buildPath = outdir + result.built = true + break + else: + result.error = "configure capable but bash executable missing" + else: + result.buildPath = outdir +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, preferredMakeType: MakeType): string = var - error = "" - if not cmakeDeps and cmakeDepStr.len != 0: - error &= &"cmake capable but {cmakeDepStr}\n" - if not conDeps and conDepStr.len != 0: - error &= &"configure capable but {conDepStr}\n" - if error.len == 0: - error = "No build files found in " & outdir - doAssert cmakeDeps or conDeps or made, &"\n# Build configuration failed - {error}\n" + lpath = findFile(lname, outdir, regex = true) + makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" + makePath = outdir + + if lpath.len != 0: + return lpath + + var buildStatus: BuildStatus + + # Simply reverse order if we want configure/make vs CMake/make + case preferredMakeType + of mtCMake: + buildStatus = buildWithCmake(makePath, cmakeFlags) + if not buildStatus.built: + buildStatus = buildWithConfMake(makePath, conFlags) + of mtConfMake: + buildStatus = buildWithConfMake(makePath, conFlags) + if not buildStatus.built: + buildStatus = buildWithCmake(makePath, cmakeFlags) + + if buildStatus.buildPath.len > 0: + buildStatus.built = findFile(lname, buildStatus.buildPath, regex = true).len > 0 + + if not buildStatus.built and fileExists(buildStatus.buildPath / "Makefile"): + make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) + buildStatus.built = true + + let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir + doAssert buildStatus.built, &"\n# Build configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) @@ -903,7 +922,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = ""): untyped = + altNames: static[string] = "", preferredMakeType: static[MakeType] = mtCMake): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. @@ -1056,7 +1075,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta when stdPath.len != 0 and stdLPath.len != 0: stdLPath else: - buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`) + buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `preferredMakeType`.MakeType) # Header path - search again in case header is generated in build `path`* = From 4156a1effa38310f86efcbe140543e7df690ef82 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 2 May 2020 21:14:19 -0600 Subject: [PATCH 103/255] Address PR issues --- nimterop/build.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index fe8f31a..882a3d3 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -5,8 +5,8 @@ import os except findExe, sleep import regex type - MakeType* = enum - mtConfMake, mtCMake + BuildType* = enum + btAutoconf, btCmake BuildStatus = object built: bool @@ -824,7 +824,7 @@ proc buildWithConfMake(outdir, flags: string): BuildStatus = else: result.buildPath = outdir -proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, preferredMakeType: MakeType): string = +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildType: BuildType): string = var lpath = findFile(lname, outdir, regex = true) makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" @@ -836,20 +836,20 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, prefer var buildStatus: BuildStatus # Simply reverse order if we want configure/make vs CMake/make - case preferredMakeType - of mtCMake: + case buildType + of btCmake: buildStatus = buildWithCmake(makePath, cmakeFlags) if not buildStatus.built: buildStatus = buildWithConfMake(makePath, conFlags) - of mtConfMake: + of btAutoconf: buildStatus = buildWithConfMake(makePath, conFlags) if not buildStatus.built: buildStatus = buildWithCmake(makePath, cmakeFlags) if buildStatus.buildPath.len > 0: - buildStatus.built = findFile(lname, buildStatus.buildPath, regex = true).len > 0 + let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 - if not buildStatus.built and fileExists(buildStatus.buildPath / "Makefile"): + if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) buildStatus.built = true @@ -922,7 +922,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", preferredMakeType: static[MakeType] = mtCMake): untyped = + altNames: static[string] = "", buildType: static[BuildType] = btCmake): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. @@ -1075,7 +1075,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta when stdPath.len != 0 and stdLPath.len != 0: stdLPath else: - buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `preferredMakeType`.MakeType) + buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildType`.BuildType) # Header path - search again in case header is generated in build `path`* = From 742fc3d96a00b4a7e90eaac26c63d68fc647cd23 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 4 May 2020 09:03:04 -0600 Subject: [PATCH 104/255] Add more maintainable build types Add buildTypes to docs --- nimterop/build.nim | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 882a3d3..ece2e33 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -810,7 +810,7 @@ proc buildWithCmake(outdir, flags: string): BuildStatus = else: result.buildPath = outdir -proc buildWithConfMake(outdir, flags: string): BuildStatus = +proc buildWithAutoConf(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if findExe("bash").len != 0: for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: @@ -824,7 +824,7 @@ proc buildWithConfMake(outdir, flags: string): BuildStatus = else: result.buildPath = outdir -proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildType: BuildType): string = +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = var lpath = findFile(lname, outdir, regex = true) makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" @@ -835,16 +835,15 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT var buildStatus: BuildStatus - # Simply reverse order if we want configure/make vs CMake/make - case buildType - of btCmake: - buildStatus = buildWithCmake(makePath, cmakeFlags) - if not buildStatus.built: - buildStatus = buildWithConfMake(makePath, conFlags) - of btAutoconf: - buildStatus = buildWithConfMake(makePath, conFlags) - if not buildStatus.built: + for buildType in buildTypes: + case buildType + of btCmake: buildStatus = buildWithCmake(makePath, cmakeFlags) + of btAutoconf: + buildStatus = buildWithAutoConf(makePath, conFlags) + + if buildStatus.built: + break if buildStatus.buildPath.len > 0: let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 @@ -922,7 +921,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", buildType: static[BuildType] = btCmake): untyped = + altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. @@ -965,6 +964,9 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## with cmake. In this case, `altNames = "z,zlib"`. Comma separate for multiple alternate names without ## spaces. ## + ## `buildTypes` specifies a list of in order build strategies to use when building the downloaded source + ## files. Default is [btCmake, btAutoconf] + ## ## The original header name is not included by default if `altNames` is set since it could cause the ## wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if `altNames = "SDL2"`. ## Explicitly include it in `altNames` like the `zlib` example when required. @@ -1075,7 +1077,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta when stdPath.len != 0 and stdLPath.len != 0: stdLPath else: - buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildType`.BuildType) + buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Header path - search again in case header is generated in build `path`* = From dc1c4bc1779e5e4bce50f566103648673e16aba8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 20:42:30 -0600 Subject: [PATCH 105/255] Add comments for enums and procs --- nimterop/ast2.nim | 36 +++++++++++++++++++++++++----------- nimterop/getters.nim | 21 ++++++++++++++++++++- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4d609cb..17fe216 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1393,14 +1393,18 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] if en.getName() == "comment": continue + let - fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField) + atom = en.getAtom() + commentNode = en.getInlineCommentNode() + fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) + if fname.nBl and gState.addNewIdentifer(fname): var fval = "" @@ -1412,9 +1416,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]), commentNode)) else: - fieldDeclarations.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode), commentNode)) fnames.incl fname prev = fname @@ -1424,18 +1428,20 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fieldDeclarations: + for (fname, fval, cexprNode, commentNode) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + constNode.comment = gState.getCommentVal(commentNode) + gState.constSection.add constNode # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode) = +proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = # Add a proc variable decho("addProcVar()") let @@ -1488,12 +1494,13 @@ proc addProcVar(gState: State, node, rnode: TSNode) = # nkEmpty() # ) + identDefs.comment = gState.getCommentVal(commentNode) # nkVarSection.add gState.varSection.add identDefs gState.printDebug(identDefs) -proc addProc(gState: State, node, rnode: TSNode) = +proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1599,6 +1606,8 @@ proc addProc(gState: State, node, rnode: TSNode) = procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) + procDef.comment = gState.getCommentVal(commentNode) + # nkProcSection.add gState.procSection.add procDef @@ -1609,18 +1618,20 @@ proc addDecl(gState: State, node: TSNode) = decho("addDecl()") gState.printDebug(node) + let start = getStartAtom(node) + commentNode = node.getCommentNode() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - gState.addProcVar(node[i], node[start]) + gState.addProcVar(node[i], node[start], commentNode) else: # proc - gState.addProc(node[i], node[start]) + gState.addProc(node[i], node[start], commentNode) else: # Regular var discard @@ -1632,11 +1643,14 @@ proc addDef(gState: State, node: TSNode) = # and will fail at link time decho("addDef()") gState.printDebug(node) + let start = getStartAtom(node) + commentNode = node.getCommentNode() + if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): - gState.addProc(node[start+1], node[start]) + gState.addProc(node[start+1], node[start], commentNode) else: gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % gState.getNodeVal(node[start+1].getAtom()) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 620c1ec..d03e65b 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times +import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options import regex @@ -456,6 +456,9 @@ proc printLisp*(gState: State, root: TSNode): string = proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") +proc getDocStrCommented*(str: string): string = + "\n## " & str.strip().replace("\n", "\n## ") + proc printTree*(gState: State, pnode: PNode, offset = ""): string = if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" @@ -634,6 +637,22 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] +proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = + if commentNode.isSome(): + result = gState.getNodeVal(commentNode.get()).replace(re"( *(/\*\*|\*\*/|\*/|\*))", "").strip() + +proc getCommentNode*(node: TSNode): Option[TSNode] = + result = none(TSNode) + let prevSibling = node.tsNodePrevNamedSibling() + if not prevSibling.isNil and prevSibling.getName() == "comment": + result = some(prevSibling) + +proc getInlineCommentNode*(node: TSNode): Option[TSNode] = + result = none(TSNode) + let nextSibling = node.tsNodeNextNamedSibling() + if not nextSibling.isNil and nextSibling.getName() == "comment": + result = some(nextSibling) + proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: for i in 0 .. node.tsNodeNamedChildCount()-1: From 5a4432bb683bc23909e575a28b1bce35b826a0ab Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 22:50:14 -0600 Subject: [PATCH 106/255] Add comments for fields and objects --- nimterop/ast2.nim | 26 ++++++++++++++++++++++---- nimterop/getters.nim | 24 ++++++++++++++---------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 17fe216..2fb48c0 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,6 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") + commentNode = node[i].getNextCommentNode() # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -716,6 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Add nkIdentDefs for each field for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: + field.comment = gState.getCommentVal(commentNode) result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = @@ -725,6 +727,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") + let commentNode = node.tsNodeParent().getPrevCommentNode() + let # Object has fields or not fdlist = node.anyChildInTree("field_declaration_list") @@ -837,6 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add + typeDef.comment = gState.getCommentVal(commentNode) gState.typeSection.add typeDef gState.printDebug(typeDef) @@ -848,6 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) + commentNode = node.getNextCommentNode() # Fix issue #185 name = @@ -859,6 +865,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" if name.nBl and gState.identifierNodes.hasKey(name): let def = gState.identifierNodes[name] + def.comment = gState.getCommentVal(commentNode) + # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and def[2].kind == nkObjectTy and def[2].len == 3 and @@ -890,6 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) + commentNode = node.getPrevCommentNode() for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -897,6 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: + typeDef.comment = gState.getCommentVal(commentNode) let name = typeDef.getIdentName() @@ -1386,7 +1396,15 @@ proc addEnum(gState: State, node: TSNode) = gState.typeSection.add eoverride elif gState.addNewIdentifer(name): # Add enum definition and helpers - gState.enumSection.add gState.parseString(&"defineEnum({name})") + let defineNode = gState.parseString(&"defineEnum({name})") + # nkStmtList( + # nkCall( + # nkIdent("defineEnum"), + # nkIdent(name) <- set the comment here + # ) + # ) + defineNode[0][1].comment = gState.getCommentVal(node.getPrevCommentNode()) + gState.enumSection.add defineNode # Create const for fields var @@ -1402,7 +1420,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNode = en.getInlineCommentNode() + commentNode = en.getNextCommentNode() fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): @@ -1621,7 +1639,7 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getCommentNode() + commentNode = node.getPrevCommentNode() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: @@ -1646,7 +1664,7 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getCommentNode() + commentNode = node.getPrevCommentNode() if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): diff --git a/nimterop/getters.nim b/nimterop/getters.nim index d03e65b..5ad9e4e 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -639,19 +639,23 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = if commentNode.isSome(): - result = gState.getNodeVal(commentNode.get()).replace(re"( *(/\*\*|\*\*/|\*/|\*))", "").strip() + result = gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").strip() -proc getCommentNode*(node: TSNode): Option[TSNode] = +template findComment(procName: untyped): untyped = result = none(TSNode) - let prevSibling = node.tsNodePrevNamedSibling() - if not prevSibling.isNil and prevSibling.getName() == "comment": - result = some(prevSibling) + var sibling = node.`procName`() + var i = 0 + while not sibling.isNil and i < maxSearch: + if sibling.getName() == "comment": + return some(sibling) + sibling = sibling.`procName`() + i += 1 -proc getInlineCommentNode*(node: TSNode): Option[TSNode] = - result = none(TSNode) - let nextSibling = node.tsNodeNextNamedSibling() - if not nextSibling.isNil and nextSibling.getName() == "comment": - result = some(nextSibling) +proc getPrevCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = + findComment(tsNodePrevNamedSibling) + +proc getNextCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = + findComment(tsNodeNextNamedSibling) proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: From f5ee979fa8238b9d165e7c46b4c715581ce032ed Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 23:41:56 -0600 Subject: [PATCH 107/255] Fix comments not being valid rst --- nimterop/getters.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 5ad9e4e..ed2cf73 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -639,7 +639,7 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = if commentNode.isSome(): - result = gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").strip() + result = "::\n " & gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").replace("\n", "\n ").strip() template findComment(procName: untyped): untyped = result = none(TSNode) @@ -651,10 +651,10 @@ template findComment(procName: untyped): untyped = sibling = sibling.`procName`() i += 1 -proc getPrevCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = +proc getPrevCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = findComment(tsNodePrevNamedSibling) -proc getNextCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = +proc getNextCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = findComment(tsNodeNextNamedSibling) proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = From b26f92b0ff2ea4be497ea91b1c16d3b9b7665c73 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 13:08:52 -0600 Subject: [PATCH 108/255] Add ability to have multiple comments --- nimterop/ast2.nim | 49 ++++++++++++++++++++++---------------------- nimterop/getters.nim | 40 ++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2fb48c0..2e601db 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,7 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") - commentNode = node[i].getNextCommentNode() + commentNodes = node[i].getNextCommentNodes() # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -717,7 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Add nkIdentDefs for each field for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: - field.comment = gState.getCommentVal(commentNode) + field.comment = gState.getCommentsStr(commentNodes) result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = @@ -727,7 +727,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") - let commentNode = node.tsNodeParent().getPrevCommentNode() + let commentNodes = node.tsNodeParent().getPrevCommentNodes() let # Object has fields or not @@ -841,7 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add - typeDef.comment = gState.getCommentVal(commentNode) + typeDef.comment = gState.getCommentsStr(commentNodes) gState.typeSection.add typeDef gState.printDebug(typeDef) @@ -853,7 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) - commentNode = node.getNextCommentNode() + commentNodes = node.getNextCommentNodes() # Fix issue #185 name = @@ -865,7 +865,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" if name.nBl and gState.identifierNodes.hasKey(name): let def = gState.identifierNodes[name] - def.comment = gState.getCommentVal(commentNode) + def.comment = gState.getCommentsStr(commentNodes) # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and @@ -898,7 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -906,7 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: - typeDef.comment = gState.getCommentVal(commentNode) + typeDef.comment = gState.getCommentsStr(commentNodes) let name = typeDef.getIdentName() @@ -1403,7 +1403,7 @@ proc addEnum(gState: State, node: TSNode) = # nkIdent(name) <- set the comment here # ) # ) - defineNode[0][1].comment = gState.getCommentVal(node.getPrevCommentNode()) + defineNode[0][1].comment = gState.getCommentsStr(node.getPrevCommentNodes()) gState.enumSection.add defineNode # Create const for fields @@ -1411,7 +1411,7 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: Option[TSNode]]] + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1420,7 +1420,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNode = en.getNextCommentNode() + commentNodes = en.getNextCommentNodes() fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): @@ -1434,9 +1434,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]), commentNode)) + fieldDeclarations.add((fname, "", some(en[1]), commentNodes)) else: - fieldDeclarations.add((fname, fval, none(TSNode), commentNode)) + fieldDeclarations.add((fname, fval, none(TSNode), commentNodes)) fnames.incl fname prev = fname @@ -1446,20 +1446,20 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode, commentNode) in fieldDeclarations: + for (fname, fval, cexprNode, commentNodes) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] - constNode.comment = gState.getCommentVal(commentNode) + constNode.comment = gState.getCommentsStr(commentNodes) gState.constSection.add constNode # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = +proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc variable decho("addProcVar()") let @@ -1512,13 +1512,13 @@ proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) # nkEmpty() # ) - identDefs.comment = gState.getCommentVal(commentNode) + identDefs.comment = gState.getCommentsStr(commentNodes) # nkVarSection.add gState.varSection.add identDefs gState.printDebug(identDefs) -proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = +proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1624,7 +1624,7 @@ proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) - procDef.comment = gState.getCommentVal(commentNode) + procDef.comment = gState.getCommentsStr(commentNodes) # nkProcSection.add gState.procSection.add procDef @@ -1636,20 +1636,19 @@ proc addDecl(gState: State, node: TSNode) = decho("addDecl()") gState.printDebug(node) - let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - gState.addProcVar(node[i], node[start], commentNode) + gState.addProcVar(node[i], node[start], commentNodes) else: # proc - gState.addProc(node[i], node[start], commentNode) + gState.addProc(node[i], node[start], commentNodes) else: # Regular var discard @@ -1664,11 +1663,11 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): - gState.addProc(node[start+1], node[start], commentNode) + gState.addProc(node[start+1], node[start], commentNodes) else: gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % gState.getNodeVal(node[start+1].getAtom()) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index ed2cf73..1240130 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,5 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options +import algorithm import regex @@ -637,12 +638,13 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] -proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = - if commentNode.isSome(): - result = "::\n " & gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").replace("\n", "\n ").strip() +proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = + if commentNodes.len > 0: + result = "::" + for commentNode in commentNodes: + result &= "\n " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() template findComment(procName: untyped): untyped = - result = none(TSNode) var sibling = node.`procName`() var i = 0 while not sibling.isNil and i < maxSearch: @@ -651,11 +653,33 @@ template findComment(procName: untyped): untyped = sibling = sibling.`procName`() i += 1 -proc getPrevCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = - findComment(tsNodePrevNamedSibling) +proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = + ## Here we want to go until the node we get is not a comment + ## for cases with multiple ``//`` comments instead of one ``/* */`` + ## section + var sibling = node.tsNodePrevNamedSibling() + var i = 0 + while not sibling.isNil and i < maxSearch: + while not sibling.isNil and sibling.getName() == "comment": + result.add(sibling) + sibling = sibling.tsNodePrevNamedSibling() + if sibling.isNil: + result.reverse + return + sibling = sibling.tsNodePrevNamedSibling() + i += 1 -proc getNextCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = - findComment(tsNodeNextNamedSibling) + result.reverse + +proc getNextCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = + ## We only want to search for the next comment node (ie: inline) + var sibling = node.tsNodeNextNamedSibling() + var i = 0 + while not sibling.isNil and i < maxSearch: + if sibling.getName() == "comment": + return @[sibling] + sibling = sibling.tsNodeNextNamedSibling() + i += 1 proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: From 4f1464bdfb0545f06edf23b2fc92a72733403a5c Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 20:47:28 -0600 Subject: [PATCH 109/255] Remove unnecessary proc --- nimterop/getters.nim | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 1240130..281e3a4 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options +import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import algorithm import regex @@ -644,15 +644,6 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() -template findComment(procName: untyped): untyped = - var sibling = node.`procName`() - var i = 0 - while not sibling.isNil and i < maxSearch: - if sibling.getName() == "comment": - return some(sibling) - sibling = sibling.`procName`() - i += 1 - proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = ## Here we want to go until the node we get is not a comment ## for cases with multiple ``//`` comments instead of one ``/* */`` From c8c2efdc3054d4f4377a41f6e1894dec4ffd0607 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 26 Apr 2020 09:28:58 -0600 Subject: [PATCH 110/255] Add comments explaining comment node procs --- nimterop/getters.nim | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 281e3a4..89fde10 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -639,6 +639,8 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool result.name = result.name[0 .. ^2] proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = + ## Generate a comment from a set of comment nodes. Comment is guaranteed + ## to be able to be rendered using nim doc if commentNodes.len > 0: result = "::" for commentNode in commentNodes: @@ -650,25 +652,37 @@ proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = ## section var sibling = node.tsNodePrevNamedSibling() var i = 0 + + # Search for the starting comment up to maxSearch nodes away while not sibling.isNil and i < maxSearch: + # Once a comment is found, find all of the comments right next to + # it so that we can get multiple // style comments while not sibling.isNil and sibling.getName() == "comment": result.add(sibling) sibling = sibling.tsNodePrevNamedSibling() + if sibling.isNil: - result.reverse - return + break + sibling = sibling.tsNodePrevNamedSibling() i += 1 + # reverse the comments because we got them in reverse order result.reverse proc getNextCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = - ## We only want to search for the next comment node (ie: inline) + ## Searches the next nodes up to maxSearch nodes away for a comment + + # We only want to search for the next comment node (ie: inline) + # but we want to keep the same interface as getPrevCommentNodes, + # so we keep a returned seq but only store one element var sibling = node.tsNodeNextNamedSibling() var i = 0 + # Search for the comment up to maxSearch nodes away while not sibling.isNil and i < maxSearch: if sibling.getName() == "comment": - return @[sibling] + result.add sibling + break sibling = sibling.tsNodeNextNamedSibling() i += 1 From b81d51dc37f316c71a29b67a0750e3ce55990b91 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 26 Apr 2020 09:56:18 -0600 Subject: [PATCH 111/255] Use gState.nocomments --- nimterop/ast2.nim | 16 ++++++++-------- nimterop/getters.nim | 12 +++++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2e601db..0fd5a9d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,7 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") - commentNodes = node[i].getNextCommentNodes() + commentNodes = gState.getNextCommentNodes(node[i]) # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -727,7 +727,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") - let commentNodes = node.tsNodeParent().getPrevCommentNodes() + let commentNodes = gState.getPrevCommentNodes(node.tsNodeParent()) let # Object has fields or not @@ -853,7 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) - commentNodes = node.getNextCommentNodes() + commentNodes = gState.getNextCommentNodes(node) # Fix issue #185 name = @@ -898,7 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) - commentNodes = node.getPrevCommentNodes() + commentNodes = gState.getPrevCommentNodes(node) for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -1403,7 +1403,7 @@ proc addEnum(gState: State, node: TSNode) = # nkIdent(name) <- set the comment here # ) # ) - defineNode[0][1].comment = gState.getCommentsStr(node.getPrevCommentNodes()) + defineNode[0][1].comment = gState.getCommentsStr(gState.getPrevCommentNodes(node)) gState.enumSection.add defineNode # Create const for fields @@ -1420,7 +1420,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNodes = en.getNextCommentNodes() + commentNodes = gState.getNextCommentNodes(en) fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): @@ -1638,7 +1638,7 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNodes = node.getPrevCommentNodes() + commentNodes = gState.getPrevCommentNodes(node) for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: @@ -1663,7 +1663,7 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNodes = node.getPrevCommentNodes() + commentNodes = gState.getPrevCommentNodes(node) if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 89fde10..148c209 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -644,12 +644,16 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = if commentNodes.len > 0: result = "::" for commentNode in commentNodes: - result &= "\n " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() + result &= "\n " & gState.getNodeVal(commentNode). + replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() -proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = +proc getPrevCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Here we want to go until the node we get is not a comment ## for cases with multiple ``//`` comments instead of one ``/* */`` ## section + if gState.nocomments: + return + var sibling = node.tsNodePrevNamedSibling() var i = 0 @@ -670,9 +674,11 @@ proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = # reverse the comments because we got them in reverse order result.reverse -proc getNextCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = +proc getNextCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Searches the next nodes up to maxSearch nodes away for a comment + if gState.nocomments: + return # We only want to search for the next comment node (ie: inline) # but we want to keep the same interface as getPrevCommentNodes, # so we keep a returned seq but only store one element From f70f836af837376bcad9a76c7fd05b930b0e7506 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 26 Apr 2020 10:47:32 -0600 Subject: [PATCH 112/255] Remove unused proc --- nimterop/getters.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 148c209..28005e3 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -457,9 +457,6 @@ proc printLisp*(gState: State, root: TSNode): string = proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") -proc getDocStrCommented*(str: string): string = - "\n## " & str.strip().replace("\n", "\n## ") - proc printTree*(gState: State, pnode: PNode, offset = ""): string = if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" From 789af2b5cc05204e1cbfda009aaee005d4153248 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 26 Apr 2020 11:07:30 -0600 Subject: [PATCH 113/255] Add comment gen for array type --- nimterop/ast2.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 0fd5a9d..ab7bb72 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1017,6 +1017,7 @@ proc addTypeArray(gState: State, node: TSNode) = # node[start] = identifier = type name (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") tident = gState.getIdent(tname, info, exported = false) + commentNodes = gState.getPrevCommentNodes(node) # Could have multiple types, comma separated for i in start+1 ..< node.len: @@ -1050,6 +1051,7 @@ proc addTypeArray(gState: State, node: TSNode) = # ) # ) + typeDef.comment = gState.getCommentsStr(commentNodes) # nkTypeSection.add gState.typeSection.add typeDef From 31bea0f032adca25bcb836fcbd7f4cb2f00d78ba Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 28 Apr 2020 22:52:24 -0600 Subject: [PATCH 114/255] Add more robust comment extraction --- nimterop/ast2.nim | 18 +++--- nimterop/getters.nim | 134 ++++++++++++++++++++++++++++++------------- 2 files changed, 102 insertions(+), 50 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index ab7bb72..b51b252 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,7 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") - commentNodes = gState.getNextCommentNodes(node[i]) + commentNodes = gState.getCommentNodes(node[i]) # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -727,7 +727,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") - let commentNodes = gState.getPrevCommentNodes(node.tsNodeParent()) + let commentNodes = gState.getCommentNodes(node.tsNodeParent()) let # Object has fields or not @@ -853,7 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) - commentNodes = gState.getNextCommentNodes(node) + commentNodes = gState.getCommentNodes(node) # Fix issue #185 name = @@ -898,7 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) - commentNodes = gState.getPrevCommentNodes(node) + commentNodes = gState.getCommentNodes(node) for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -1017,7 +1017,7 @@ proc addTypeArray(gState: State, node: TSNode) = # node[start] = identifier = type name (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") tident = gState.getIdent(tname, info, exported = false) - commentNodes = gState.getPrevCommentNodes(node) + commentNodes = gState.getCommentNodes(node) # Could have multiple types, comma separated for i in start+1 ..< node.len: @@ -1405,7 +1405,7 @@ proc addEnum(gState: State, node: TSNode) = # nkIdent(name) <- set the comment here # ) # ) - defineNode[0][1].comment = gState.getCommentsStr(gState.getPrevCommentNodes(node)) + defineNode[0][1].comment = gState.getCommentsStr(gState.getCommentNodes(node)) gState.enumSection.add defineNode # Create const for fields @@ -1422,7 +1422,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNodes = gState.getNextCommentNodes(en) + commentNodes = gState.getCommentNodes(en) fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): @@ -1640,7 +1640,7 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNodes = gState.getPrevCommentNodes(node) + commentNodes = gState.getCommentNodes(node) for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: @@ -1665,7 +1665,7 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNodes = gState.getPrevCommentNodes(node) + commentNodes = gState.getCommentNodes(node) if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 28005e3..05d9448 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,5 +1,4 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times -import algorithm import regex @@ -572,30 +571,30 @@ proc getPreprocessor*(gState: State, fullpath: string): string = # Include content only from file for line in execAction(cmd).output.splitLines(): - if line.strip() != "": - if line.len > 1 and line[0 .. 1] == "# ": - start = false + # We want to keep blank lines here for comment processing + if line.len > 1 and line[0 .. 1] == "# ": + start = false + let + saniLine = line.sanitizePath(noQuote = true) + if sfile in saniLine: + start = true + elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + start = true + elif gState.recurse: let - saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine: + pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) + if pDir.Bl or pDir in saniLine: start = true - elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: - start = true - elif gState.recurse: - let - pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) - if pDir.Bl or pDir in saniLine: - start = true - else: - for inc in gState.includeDirs: - if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: - start = true - break - else: - if start: - if "#undef" in line: - continue - rdata.add line + else: + for inc in gState.includeDirs: + if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: + start = true + break + else: + if start: + if "#undef" in line: + continue + rdata.add line return rdata.join("\n") converter toString*(kind: Kind): string = @@ -644,32 +643,85 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = result &= "\n " & gState.getNodeVal(commentNode). replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() -proc getPrevCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = - ## Here we want to go until the node we get is not a comment - ## for cases with multiple ``//`` comments instead of one ``/* */`` - ## section +proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = + ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` + ## nodes before and after the current node + ## + ## Priority is (closest line number) > comment before > comment after. + ## This priority might need to be changed based on the project, but + ## for now it is good enough + + # Skip this if we don't want comments if gState.nocomments: return - var sibling = node.tsNodePrevNamedSibling() - var i = 0 + let (line, _) = gState.getLineCol(node) - # Search for the starting comment up to maxSearch nodes away - while not sibling.isNil and i < maxSearch: - # Once a comment is found, find all of the comments right next to - # it so that we can get multiple // style comments - while not sibling.isNil and sibling.getName() == "comment": - result.add(sibling) - sibling = sibling.tsNodePrevNamedSibling() + # Keep track of both directions from a node + var + prevSibling = node.tsNodePrevNamedSibling() + nextSibling = node.tsNodeNextNamedSibling() + nilNode: TSNode - if sibling.isNil: + var + i = 0 + prevSiblingDistance, nextSiblingDistance: int + lowestDistance: int + commentsFound = false + + while not commentsFound and i < maxSearch: + + # Distance from the current node will tell us approximately if the + # comment belongs to the node. The closer it is in terms of line + # numbers, the more we can be sure it's the comment we want + if not prevSibling.isNil: + prevSiblingDistance = abs(gState.getLineCol(prevSibling)[0] - line) + if not nextSibling.isNil: + nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) + + lowestDistance = min(prevSiblingDistance, nextSiblingDistance) + + if prevSiblingDistance > maxSearch: + # If the line is out of range, skip searching + prevSibling = nilNode # Can't do `= nil` + + if nextSiblingDistance > maxSearch: + # If the line is out of range, skip searching + prevSibling = nilNode + + while ( + not prevSibling.isNil and + prevSibling.getName() == "comment" and + prevSiblingDistance == lowestDistance + ): + # Put the previous nodes in reverse order so the comments + # make logical sense + result.insert(prevSibling, 0) + prevSibling = prevSibling.tsNodePrevNamedSibling() + commentsFound = true + + if commentsFound: break - sibling = sibling.tsNodePrevNamedSibling() - i += 1 + while ( + not nextSibling.isNil and + nextSibling.getName() == "comment" and + nextSiblingDistance == lowestDistance + ): + result.add(nextSibling) + nextSibling = nextSibling.tsNodeNextNamedSibling() + commentsFound = true - # reverse the comments because we got them in reverse order - result.reverse + if commentsFound: + break + + # Go to next sibling pair + if not prevSibling.isNil: + prevSibling = prevSibling.tsNodePrevNamedSibling() + if not nextSibling.isNil: + nextSibling = nextSibling.tsNodeNextNamedSibling() + + i += 1 proc getNextCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Searches the next nodes up to maxSearch nodes away for a comment From 1e105d34a49469b9c37ecbd38032765dbef96049 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 29 Apr 2020 19:15:11 -0600 Subject: [PATCH 115/255] Add docgen runs in tests Fix paths import Skip docgen for rsa for now --- nimterop.nimble | 13 ++++++++++--- nimterop/cimport.nim | 2 +- nimterop/docs.nim | 4 ++-- tests/tnimterop_c.nim | 4 +++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index a6ece7c..8a47465 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -12,14 +12,21 @@ installDirs = @["nimterop"] requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" import nimterop/docs +import os proc execCmd(cmd: string) = exec "tests/timeit " & cmd -proc execTest(test: string, flags = "") = +proc execTest(test: string, flags = "", runDocs = true) = execCmd "nim c --hints:off -f " & flags & " -r " & test execCmd "nim cpp --hints:off " & flags & " -r " & test + if runDocs: + let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs" + rmDir docPath + mkDir docPath + buildDocs(@[test], docPath, compilerArgs = flags) + task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") @@ -54,8 +61,8 @@ task test, "Test": execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" when defined(Linux): - execTest "tests/rsa.nim" - execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"" + execTest "tests/rsa.nim", runDocs = false + execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"", runDocs = false # Platform specific tests when defined(Windows): diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index e081633..99be1f5 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -434,7 +434,7 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to ## `cSearchPath() `_. runnableExamples: - import paths, os + import nimterop/paths, os static: cAddSearchDir testsIncludeDir() doAssert cSearchPath("test.h").existsFile diff --git a/nimterop/docs.nim b/nimterop/docs.nim index d808535..e0fef39 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -36,7 +36,7 @@ proc execAction(cmd: string): string = doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep, - defines: openArray[string] = @[]) = + defines: openArray[string] = @[], compilerArgs = "") = ## Generate docs for all specified nim `files` to the specified `path` ## ## `baseDir` is the project path by default and `files` and `path` are relative @@ -70,7 +70,7 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath defStr nim = getCurrentCompilerExe() for file in files: - echo execAction(&"{nim} doc {defStr} -o:{path} --project --index:on {baseDir & file}") + echo execAction(&"{nim} doc {defStr} {compilerArgs} -o:{path} --project --index:on {baseDir & file}") echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}") when declared(getNimRootDir): diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index 20c1678..ef814ea 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -1,4 +1,6 @@ import std/unittest +import os +import macros import nimterop/cimport import nimterop/paths @@ -10,7 +12,7 @@ cDefine("FORCE") cIncludeDir testsIncludeDir() cCompile cSearchPath("test.c") -cPluginPath("tests/tnimterop_c_plugin.nim") +cPluginPath(getProjectPath() / "tnimterop_c_plugin.nim") cOverride: type From f5ddcb7f548d3291f1ca64760587f5d85904e2ec Mon Sep 17 00:00:00 2001 From: Joey Date: Thu, 30 Apr 2020 08:43:40 -0600 Subject: [PATCH 116/255] Change install to develop for appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5275844..d67b01d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ for: - /home/appveyor/binaries build_script: - - nimble --verbose install -y + - nimble --verbose develop -y test_script: - nimble --verbose test From 0d65bcdd52d7eda9d4e642d2ad7bb0e3f5a97e0a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 30 Apr 2020 11:55:07 -0600 Subject: [PATCH 117/255] Run docs on rsa.nim --- nimterop.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8a47465..38a2d5d 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -61,8 +61,8 @@ task test, "Test": execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" when defined(Linux): - execTest "tests/rsa.nim", runDocs = false - execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"", runDocs = false + execTest "tests/rsa.nim" + execTest "tests/rsa.nim", "-d:FLAGS=\"-H\"" # Platform specific tests when defined(Windows): From 964209d5f855f5933f5d5aa8b4d18fa5fa1fbc03 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 30 Apr 2020 12:45:00 -0600 Subject: [PATCH 118/255] Remove unneeded proc, add more comments --- nimterop/getters.nim | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 05d9448..cae2d08 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -670,7 +670,6 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = commentsFound = false while not commentsFound and i < maxSearch: - # Distance from the current node will tell us approximately if the # comment belongs to the node. The closer it is in terms of line # numbers, the more we can be sure it's the comment we want @@ -689,6 +688,9 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = # If the line is out of range, skip searching prevSibling = nilNode + # Search above the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments while ( not prevSibling.isNil and prevSibling.getName() == "comment" and @@ -700,9 +702,13 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = prevSibling = prevSibling.tsNodePrevNamedSibling() commentsFound = true + # If we've already found comments above the current line, quit if commentsFound: break + # Search below or at the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments while ( not nextSibling.isNil and nextSibling.getName() == "comment" and @@ -723,24 +729,6 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = i += 1 -proc getNextCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = - ## Searches the next nodes up to maxSearch nodes away for a comment - - if gState.nocomments: - return - # We only want to search for the next comment node (ie: inline) - # but we want to keep the same interface as getPrevCommentNodes, - # so we keep a returned seq but only store one element - var sibling = node.tsNodeNextNamedSibling() - var i = 0 - # Search for the comment up to maxSearch nodes away - while not sibling.isNil and i < maxSearch: - if sibling.getName() == "comment": - result.add sibling - break - sibling = sibling.tsNodeNextNamedSibling() - i += 1 - proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: for i in 0 .. node.tsNodeNamedChildCount()-1: From 461dde150ee5fe4350d593e5ca675b1a9438d534 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 30 Apr 2020 17:23:10 -0600 Subject: [PATCH 119/255] Add escaping for rst --- nimterop/getters.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index cae2d08..099faf4 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -638,9 +638,10 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = ## Generate a comment from a set of comment nodes. Comment is guaranteed ## to be able to be rendered using nim doc if commentNodes.len > 0: + const escapeRstReg = re"""(["!#$%&'()*+,-./:;<=>?@[\]^_`{|}~])""" result = "::" for commentNode in commentNodes: - result &= "\n " & gState.getNodeVal(commentNode). + result &= "\n " & gState.getNodeVal(commentNode).replace(escapeRstReg, r"\$1"). replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = From 5e7ec3d88c7cf817ec716f4b24e9564906f258f6 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 30 Apr 2020 19:25:13 -0600 Subject: [PATCH 120/255] Fix bug with comment generation --- nimterop/getters.nim | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 099faf4..10c3b5e 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -387,6 +387,16 @@ proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = getLineCol(gState.code, node) +proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node + let + point = node.tsNodeEndPoint() + result.line = point.row.int + 1 + result.col = point.column.int + 1 + +proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getEndLineCol(gState.code, node) + proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = for i in 0 ..< node.len: if node.getName() != "comment": @@ -638,11 +648,12 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = ## Generate a comment from a set of comment nodes. Comment is guaranteed ## to be able to be rendered using nim doc if commentNodes.len > 0: - const escapeRstReg = re"""(["!#$%&'()*+,-./:;<=>?@[\]^_`{|}~])""" result = "::" for commentNode in commentNodes: - result &= "\n " & gState.getNodeVal(commentNode).replace(escapeRstReg, r"\$1"). - replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() + result &= "\n " & gState.getNodeVal(commentNode).strip() + + result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") + result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` @@ -666,7 +677,7 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = var i = 0 - prevSiblingDistance, nextSiblingDistance: int + prevSiblingDistance, nextSiblingDistance: int = int.high lowestDistance: int commentsFound = false @@ -675,9 +686,15 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = # comment belongs to the node. The closer it is in terms of line # numbers, the more we can be sure it's the comment we want if not prevSibling.isNil: - prevSiblingDistance = abs(gState.getLineCol(prevSibling)[0] - line) + if prevSibling.getName() == "comment": + prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line) + else: + prevSiblingDistance = int.high if not nextSibling.isNil: - nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) + if nextSibling.getName() == "comment": + nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) + else: + nextSiblingDistance = int.high lowestDistance = min(prevSiblingDistance, nextSiblingDistance) @@ -687,7 +704,7 @@ proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = if nextSiblingDistance > maxSearch: # If the line is out of range, skip searching - prevSibling = nilNode + nextSibling = nilNode # Search above the current line for comments. When one is found # keep going to retrieve successive comments for cases with multiple From 5104eeb5c0da96509c621202ea9e94c371e20933 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 3 May 2020 18:40:49 -0600 Subject: [PATCH 121/255] compilerArgs -> nimArgs, fix multi decl comments --- nimterop.nimble | 2 +- nimterop/ast2.nim | 17 ++++++++++++++++- nimterop/docs.nim | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 38a2d5d..702c714 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -25,7 +25,7 @@ proc execTest(test: string, flags = "", runDocs = true) = let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs" rmDir docPath mkDir docPath - buildDocs(@[test], docPath, compilerArgs = flags) + buildDocs(@[test], docPath, nimArgs = flags) task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index b51b252..759419c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1640,16 +1640,31 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNodes = gState.getCommentNodes(node) + + var + firstDecl = true + commentNodes: seq[TSNode] for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var + if firstDecl: + # If + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) gState.addProcVar(node[i], node[start], commentNodes) else: # proc + if firstDecl: + # If + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) gState.addProc(node[i], node[start], commentNodes) else: # Regular var diff --git a/nimterop/docs.nim b/nimterop/docs.nim index e0fef39..798df52 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -36,7 +36,7 @@ proc execAction(cmd: string): string = doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep, - defines: openArray[string] = @[], compilerArgs = "") = + defines: openArray[string] = @[], nimArgs = "") = ## Generate docs for all specified nim `files` to the specified `path` ## ## `baseDir` is the project path by default and `files` and `path` are relative @@ -70,7 +70,7 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath defStr nim = getCurrentCompilerExe() for file in files: - echo execAction(&"{nim} doc {defStr} {compilerArgs} -o:{path} --project --index:on {baseDir & file}") + echo execAction(&"{nim} doc {defStr} {nimArgs} -o:{path} --project --index:on {baseDir & file}") echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}") when declared(getNimRootDir): From 36c7331749a3f2ef87d64edf8dcaed4b79d434fb Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 4 May 2020 08:07:36 -0600 Subject: [PATCH 122/255] Add nimArgs to buildDocs docstring --- nimterop/docs.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 798df52..d5bd50c 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -45,6 +45,8 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath ## `defines` is a list of `-d:xxx` define flags (the `xxx` part) that should be passed ## to `nim doc` so that `getHeader()` is invoked correctly. ## + ## `nimArgs` is a string representing extra arguments to send to the `nim doc` call. + ## ## Use the `--publish` flag with nimble to publish docs contained in ## `path` to Github in the `gh-pages` branch. This requires the ghp-import ## package for Python: `pip install ghp-import` From 25660ac47532e4a6e0943e20f42ee34b1165a690 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 4 May 2020 08:10:46 -0600 Subject: [PATCH 123/255] Fix missing comment in ast2 --- nimterop/ast2.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 759419c..fd70b46 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1651,7 +1651,8 @@ proc addDecl(gState: State, node: TSNode) = if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var if firstDecl: - # If + # If it's the first declaration, use the whole node + # to get the comment above/below commentNodes = gState.getCommentNodes(node) firstDecl = false else: @@ -1660,7 +1661,8 @@ proc addDecl(gState: State, node: TSNode) = else: # proc if firstDecl: - # If + # If it's the first declaration, use the whole node + # to get the comment above/below commentNodes = gState.getCommentNodes(node) firstDecl = false else: From 1f2d7892adc6e63443526bbc263af644bf91893c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 5 May 2020 11:04:17 -0500 Subject: [PATCH 124/255] Enable -d:checkAbi in tests --- nimterop.nimble | 7 +++++-- tests/getheader.nims | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 702c714..bd4521d 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -18,8 +18,11 @@ proc execCmd(cmd: string) = exec "tests/timeit " & cmd proc execTest(test: string, flags = "", runDocs = true) = - execCmd "nim c --hints:off -f " & flags & " -r " & test - execCmd "nim cpp --hints:off " & flags & " -r " & test + execCmd "nim c --hints:off -f -d:checkAbi " & flags & " -r " & test + let + # -d:checkAbi broken in cpp mode until post 1.2.0 + cppAbi = when (NimMajor, NimMinor) >= (1, 3): "-d:checkAbi " else: "" + execCmd "nim cpp --hints:off " & cppAbi & flags & " -r " & test if runDocs: let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs" diff --git a/tests/getheader.nims b/tests/getheader.nims index 987545a..be22967 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -14,7 +14,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = doAssert outp.contains(output), outp var - cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\"" + cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\" -d:checkAbi" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" lexp = "liblzma version = " From 3fe6e51780d9d5c789d83060793643c7d9c1ca8d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 5 May 2020 12:23:16 -0500 Subject: [PATCH 125/255] Fix tast2.h --- nimterop.nimble | 2 +- tests/include/tast2.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index bd4521d..984d0c5 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -28,7 +28,7 @@ proc execTest(test: string, flags = "", runDocs = true) = let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs" rmDir docPath mkDir docPath - buildDocs(@[test], docPath, nimArgs = flags) + buildDocs(@[test], docPath, nimArgs = "--hints:off " & flags) task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e1d4529..2ddb58e 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -75,7 +75,7 @@ typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); -typedef int A13(int, int, void (*func)(void)); +typedef int (*A13)(int, int, void (*func)(void)); struct A14 { volatile char a1; }; struct A15 { char *a1; const int *a2[1]; }; From 87eef11ec17713c0eb6f5f17c419216efde79fa8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 5 May 2020 14:47:42 -0600 Subject: [PATCH 126/255] Fix pointer issue and test --- nimterop/exprparser.nim | 10 +++++++++- tests/include/tast2.h | 1 + tests/tast2.nim | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e68c0b1..e85310a 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -548,7 +548,15 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input -> true, false # Output -> true, false result = gState.parseString(node.val) - of "type_descriptor", "sized_type_specifier": + of "type_descriptor": + let pointerDecl = node.anyChildInTree("abstract_pointer_declarator") + if pointerDecl.isNil: + result = gState.processTSNode(node[0], typeofNode) + else: + result = nkPtrTy.newTree( + gState.processTSNode(node[0], typeofNode) + ) + of "sized_type_specifier", "primitive_type", "type_identifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc let ty = getType(node.val) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 2ddb58e..c7b85c7 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -20,6 +20,7 @@ extern "C" { #define COERCE 645635634896ull + 35436 #define COERCE2 645635634896 + 35436ul #define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define POINTEREXPR (int*)0 #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) diff --git a/tests/tast2.nim b/tests/tast2.nim index 54b89e5..4254a9d 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -150,6 +150,8 @@ assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) assert SHL3 == (1.uint shl 3) +assert typeof(POINTEREXPR) is (ptr cint) + assert ALLSHL == (SHL1 or SHL2 or SHL3) assert A0 is object From d5c15d0d3aebc1eced79faa0c1621351be56f60a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 5 May 2020 20:36:02 -0600 Subject: [PATCH 127/255] Add multiple pointer support --- nimterop/ast2.nim | 48 --------------------------------------- nimterop/comphelp.nim | 50 ++++++++++++++++++++++++++++++++++++++++- nimterop/exprparser.nim | 6 ++--- tests/include/tast2.h | 1 + tests/tast2.nim | 1 + 5 files changed, 54 insertions(+), 52 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index fd70b46..4c06cc6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -8,17 +8,6 @@ import "."/treesitter/api import "."/[comphelp, exprparser, globals, getters, tshelp] -proc getPtrType*(str: string): string = - result = case str: - of "cchar": - "cstring" - of "object": - "pointer" - of "FILE": - "File" - else: - str - proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined # and use that if present @@ -405,43 +394,6 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: else: gecho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) -proc newPtrTree(gState: State, count: int, typ: PNode): PNode = - # Create nkPtrTy tree depending on count - # - # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring - result = typ - var - count = count - if typ.kind == nkIdent: - let - tname = typ.ident.s - ptname = getPtrType(tname) - if tname != ptname: - # If Nim type available, use that ident - result = gState.getIdent(ptname, typ.info, exported = false) - # One ptr reduced - count -= 1 - if count > 0: - # Nested nkPtrTy(typ) depending on count - # - # [ptr ...] typ - # - # nkPtrTy( - # nkPtrTy( - # typ - # ) - # ) - var - nresult = newNode(nkPtrTy) - parent = nresult - child: PNode - for i in 1 ..< count: - child = newNode(nkPtrTy) - parent.add child - parent = child - parent.add result - result = nresult - proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input # diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim index 1709f8b..18915b0 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/comphelp.nim @@ -15,4 +15,52 @@ proc parseString*(gState: State, str: string): PNode = str, gState.identCache, gState.config, errorHandler = handleError ) except: - decho getCurrentExceptionMsg() \ No newline at end of file + decho getCurrentExceptionMsg() + +proc getPtrType*(str: string): string = + result = case str: + of "cchar": + "cstring" + of "object": + "pointer" + of "FILE": + "File" + else: + str + +proc newPtrTree*(gState: State, count: int, typ: PNode): PNode = + # Create nkPtrTy tree depending on count + # + # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring + result = typ + var + count = count + if typ.kind == nkIdent: + let + tname = typ.ident.s + ptname = getPtrType(tname) + if tname != ptname: + # If Nim type available, use that ident + result = gState.getIdent(ptname, typ.info, exported = false) + # One ptr reduced + count -= 1 + if count > 0: + # Nested nkPtrTy(typ) depending on count + # + # [ptr ...] typ + # + # nkPtrTy( + # nkPtrTy( + # typ + # ) + # ) + var + nresult = newNode(nkPtrTy) + parent = nresult + child: PNode + for i in 1 ..< count: + child = newNode(nkPtrTy) + parent.add child + parent = child + parent.add result + result = nresult \ No newline at end of file diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e85310a..5372c2f 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -550,12 +550,12 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.parseString(node.val) of "type_descriptor": let pointerDecl = node.anyChildInTree("abstract_pointer_declarator") + if pointerDecl.isNil: result = gState.processTSNode(node[0], typeofNode) else: - result = nkPtrTy.newTree( - gState.processTSNode(node[0], typeofNode) - ) + let pointerCount = pointerDecl.getXCount("abstract_pointer_declarator") + result = gState.newPtrTree(pointerCount, gState.processTSNode(node[0], typeofNode)) of "sized_type_specifier", "primitive_type", "type_identifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc diff --git a/tests/include/tast2.h b/tests/include/tast2.h index c7b85c7..0b1fca2 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -21,6 +21,7 @@ extern "C" { #define COERCE2 645635634896 + 35436ul #define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) #define POINTEREXPR (int*)0 +#define POINTERPOINTERPOINTEREXPR (int***)0 #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) diff --git a/tests/tast2.nim b/tests/tast2.nim index 4254a9d..6cdd606 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -151,6 +151,7 @@ assert SHL2 == (1.uint shl 2) assert SHL3 == (1.uint shl 3) assert typeof(POINTEREXPR) is (ptr cint) +assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) From 4ffb49a531124d6219f594eff3eb6e192528d27a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 6 May 2020 07:20:29 -0600 Subject: [PATCH 128/255] Add comment, potentially fix linux test --- nimterop/ast2.nim | 2 +- nimterop/exprparser.nim | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4c06cc6..aa0e8d8 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -106,7 +106,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if not maybeTyNode.isNil: let name = maybeTyNode.getName() case name - of "type_descriptor", "sized_type_specifier": + of "type_descriptor", "sized_type_specifier", "primitive_type": discard else: # Can't do gState.parseCExpression(root) here for some reason? diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 5372c2f..b0c34c1 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -549,6 +549,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Output -> true, false result = gState.parseString(node.val) of "type_descriptor": + # Input => int* + # (type_descriptor 1 2 4 "int*" + # (type_identifier 1 2 3 "int") + # (abstract_pointer_declarator 1 3 1 "*") + # ) + # + # Output => ptr int + # + # nkPtrTy( + # nkIdent("int") + # ) let pointerDecl = node.anyChildInTree("abstract_pointer_declarator") if pointerDecl.isNil: From c7060f4c5598228585f35057b57bc7435c7f09ce Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 6 May 2020 07:21:34 -0600 Subject: [PATCH 129/255] Add type identifier as ignored too --- nimterop/ast2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index aa0e8d8..6f05a34 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -106,7 +106,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if not maybeTyNode.isNil: let name = maybeTyNode.getName() case name - of "type_descriptor", "sized_type_specifier", "primitive_type": + of "type_descriptor", "sized_type_specifier", "primitive_type", "type_identifier": discard else: # Can't do gState.parseCExpression(root) here for some reason? From fabb46a639ad38c349e22f5fc0ab37b187772ec3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 6 May 2020 13:06:18 -0500 Subject: [PATCH 130/255] Set nimconf paths to absolute --- nimterop/nimconf.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 144a7e0..dca68a8 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -95,6 +95,13 @@ proc jsonToSeq(node: JsonNode, key: string): seq[string] = for elem in node[key].getElems(): result.add elem.getStr() +proc getAbsoluteDir(projectDir, path: string): string = + # Path is relative to `projectDir` if not absolute + if path.isAbsolute(): + result = path + else: + result = (projectDir / path).normalizedPath() + proc getNimConfig*(projectDir = ""): Config = # Get `paths` - list of paths to be forwarded to Nim result = new(Config) @@ -146,12 +153,15 @@ proc getNimConfig*(projectDir = ""): Config = # Find non standard lib paths added to `searchPath` for path in searchPaths: + let + path = getAbsoluteDir(projectDir, path) if libPath notin path: result.paths.incl path # Find `nimblePaths` in `lazyPaths` for path in lazyPaths: let + path = getAbsoluteDir(projectDir, path) (_, tail) = path.strip(leading = false, chars = {'/', '\\'}).splitPath() if tail == "pkgs": # Nimble path probably @@ -161,6 +171,8 @@ proc getNimConfig*(projectDir = ""): Config = # Have to do this separately since `nimblePaths` could be after # packages in `lazyPaths` for path in lazyPaths: + let + path = getAbsoluteDir(projectDir, path) var skip = false for npath in result.nimblePaths: if npath in path: From 5a6eb043d28fb03b19c14ce766f166f59fe269ab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 6 May 2020 15:02:23 -0500 Subject: [PATCH 131/255] global.nim cleanup --- nimterop/globals.nim | 105 +++++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index ae9a6d0..21214f3 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -43,6 +43,39 @@ type zeroOrOne # ? orWithNext # ! + PragmaMode* = enum + # includeHeader = true, dynlib = false + # + # Types = {.header, .importc.} + # Static inline = {.header, .importc.} + # Procs = {.header, .importc.} + # Lib = {.compile.} or {.passL.} + useHeader + + # includeHeader = true, dynlib = true + # + # Types = {.header, .importc.} + # Static inline = {.header, .importc.} + # Procs = {.dynlib, importc.} + # Libs = loaded by dynlib + useHeaderDynlib + + # includeHeader = false, dynlib = false + # + # Types = {.bycopy.} + # Static inline = not available + # Procs = {.importc.} + # Lib = {.compile.} or {.passL.} + noHeader + + # includeHeader = false, dynlib = true + # + # Types = {.bycopy.} + # Static inline = not available + # Procs = {.dynlib, importc.} + # Libs = loaded by dynlib + noHeaderDynlib + Ast* = object name*: string kind*: Kind @@ -55,37 +88,51 @@ type AstTable* {.used.} = TableRef[string, seq[ref Ast]] State* = ref object - compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string] - - debug*, includeHeader*, nocache*, nocomments*, past*, preprocess*, pnim*, recurse*: bool - - code*, convention*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string - + # Command line arguments to toast - some forwarded from cimport.nim + convention*: string # `--convention | -C` to change calling convention from cdecl default + debug*: bool # `cDebug()` or `--debug | -d` to enable debug mode + defines*: seq[string] # Symbols added by `cDefine()` and `--define | -D` for C/C++ preprocessor/compiler + dynlib*: string # `cImport(dynlib)` or `--dynlib | -l` to specify variable containing library name + feature*: seq[Feature] # `--feature | -f` feature flags enabled + includeDirs*: seq[string] # Paths added by `cIncludeDir()` and `--includeDirs | -I` for C/C++ preprocessor/compiler + includeHeader*: bool # `--includeHeader | -H` to include {.header.} pragma to wrapper + mode*: string # `cImport(mode)` or `--mode | -m` to override detected compiler mode - c or cpp + nim*: string # `--nim` to specify full path to Nim compiler + nocomments*: bool # `--nocomments | -c` to disable rendering comments in wrappers + past*: bool # `--past | -a` to print tree-sitter AST of code + pluginSourcePath*: string # `--pluginSourcePath` specified path to plugin file to compile and load + pnim*: bool # `--pnim | -n` to render Nim wrapper for header + preprocess*: bool # `--preprocess | -p` to enable preprocessing of code before wrapping + prefix*: seq[string] # `--prefix` strings to strip from start of identifiers + recurse*: bool # `--recurse | -r` to recurse into #include files in headers specified replace*: OrderedTableRef[string, string] + # `--replace | -G` replacement rules for identifiers + suffix*: seq[string] # `--suffix` strings to strip from end of identifiers + symOverride*: seq[string] # `cSkipSymbol()`, `cOverride()` and `--symOverride | -O` symbols to skip during wrapping - feature*: seq[Feature] + # cimport.nim specific + compile*: seq[string] # `cCompile()` list of files already processed + nocache*: bool # `cDisableCaching()` to disable caching of artifacts + overrides*: string # `cOverride()` code which gets added to `cPlugin()` output + pluginSource*: string # `cPlugin()` generated code to write to plugin file from + searchDirs*: seq[string] # `cSearchPath()` added directories for header search + # Data fields + code*: string # Contents of header file currently being processed + currentHeader*: string # Const name of header being currently processed + impShort*: string # Short base name for pragma in output + outputHandle*: File # `--output | -o` open file handle + sourceFile*: string # Full path of header being currently processed + + # Plugin callbacks onSymbol*, onSymbolOverride*: OnSymbol onSymbolOverrideFinal*: OnSymbolOverrideFinal - outputHandle*: File - - # All symbols that have been declared so far indexed by nimName - identifiers*: TableRef[string, string] - - # All const names for enum casting - constIdentifiers*: HashSet[string] - - # All symbols that have been skipped due to - # being unwrappable or the user provided - # override is blank - skippedSyms*: HashSet[string] - - # Legacy ast fields, remove when ast2 becomes default - constStr*, enumStr*, procStr*, typeStr*: string - - commentStr*, debugStr*, skipStr*: string - + # Symbol tables + constIdentifiers*: HashSet[string] # Const names for enum casting + identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName + skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or + # the user provided override is blank # Nim compiler objects when defined(TOAST): constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode @@ -93,16 +140,16 @@ type config*: ConfigRef graph*: ModuleGraph - # Craeted symbols to generated AST - forward declaration tracking + # Table of symbols to generated AST PNode - used to implement forward declarations identifierNodes*: TableRef[string, PNode] - currentHeader*, impShort*, sourceFile*: string - # Used for the exprparser.nim module currentExpr*, currentTyCastName*: string + # Legacy AST fields, remove when ast2 becomes default + constStr*, enumStr*, procStr*, typeStr*: string + commentStr*, debugStr*, skipStr*: string data*: seq[tuple[name, val: string]] - nodeBranch*: seq[string] Feature* = enum From 701a8793c2a30dfcecdf24023be3c8091e724d2c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 9 May 2020 02:34:59 -0500 Subject: [PATCH 132/255] Fix #202 - implement noHeader --- CHANGES.md | 8 +- README.md | 61 ++++-- nimterop.nimble | 4 +- nimterop/ast.nim | 6 +- nimterop/ast2.nim | 105 +++++----- nimterop/comphelp.nim | 84 +++++++- nimterop/docs.nim | 4 +- nimterop/getters.nim | 459 ++---------------------------------------- nimterop/globals.nim | 37 +--- nimterop/grammar.nim | 8 +- nimterop/toast.nim | 24 +-- nimterop/tshelp.nim | 354 +++++++++++++++++++++++++++++++- tests/include/tast2.h | 2 +- tests/tast2.nim | 18 +- tests/tpcre.nim | 2 + 15 files changed, 583 insertions(+), 593 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b02453c..ef66c79 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,18 +14,18 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 ### Breaking changes -- Nimterop now skips generating the `{.header.}` pragma by default in non-dynlib mode. This skips the header file `#include` in the generated code and allows creation of wrappers that do not require presence of the header during compile time. There are cases where this will not work so the `--includeHeader | -H` flag is available to revert to the legacy behavior when required. This change applies to both `ast2` and the legacy backend so if an existing wrapper breaks, test it with `-H` to see if it starts working again. [#169][i169] - -- Nimterop defaulted to C++ mode for preprocessing and tree-sitter parsing in all cases unless explicitly informed to use C mode. This has been changed and is now detected based on the file extension. This means some existing wrappers could break since they might contain C++ code or include C++ headers like `#include ` which will not work in C mode. Explicitly setting `mode = "cpp"` or `-mcpp` should fix such issues. [#176][i176] +- Nimterop used to default to C++ mode for preprocessing and tree-sitter parsing in all cases unless explicitly informed to use C mode. This has been changed and is now detected based on the file extension. This means some existing wrappers could break since they might contain C++ code or include C++ headers like `#include ` which will not work in C mode. Explicitly setting `mode = "cpp"` or `-mcpp` should fix such issues. [#176][i176] - Enums were originally being mapped to `distint int` - this has been changed to `distinct cint` since the sizes are incorrect on 64-bit and is especially noticeable when types or unions have enum fields. -- `static inline` functions are no longer wrapped by the legacy backend. The `ast2` backend correctly generates wrappers for such functions but they are only generated when `--includeHeader | -H` is in effect. This is because such functions do not exist in the binary and can only be referenced when the header is compiled in. +- `static inline` functions are no longer wrapped by the legacy backend. The `ast2` backend correctly generates wrappers for such functions but they are only generated when `--noHeader | -H` is not in effect. This is because such functions do not exist in the binary and can only be referenced when the header is compiled in. - Support for Nim v0.19.6 has been dropped and the test matrix now covers v0.20.2, v1.0.6, v1.2.0 and devel. ### New functionality +- Nimterop can now skip generating the `{.header.}` pragma when the `--noHeader | -H` flag is used. This skips the header file `#include` in the generated code and allows creation of wrappers that do not require presence of the header during compile time. Note that `static inline` functions will only be wrapped when the header is compiled in. This change applies to both `ast2` and the legacy backend, although `ast2` can also generate wrappers with both `{.header.}` and `{.dynlib.}` in effect enabling type size checking with `-d:checkAbi`. More information is available in the [README.md](README.md). [#169][i169] + - `ast2` includes support for various C constructs that were issues with the legacy backend. These changes should reduce the reliance on `cOverride()` and existing wrappers should attempt to clean up such sections where possible. - N-dimensional arrays and pointers - [#54][i54] - Synomyms for types - [#74][i74] diff --git a/README.md b/README.md index 8680048..869ac4e 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ else: Module documentation for the build API can be found [here](https://nimterop.github.io/nimterop/build.html). Refer to the ```tests``` directory for additional examples on how the library can be used. Also, check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. They will provide real world examples of how to wrap libraries. Please do add your project once you are done so that others can benefit from your work. -__Download / Search__ +#### Download / Search The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. @@ -78,19 +78,19 @@ The above wrapper is generic and allows the end user to control how it works. No - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL if defined. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. -__Pre build__ +#### Pre build `getHeader()` provides a `headerPreBuild()` hook that gets called after the library is downloaded but before it is built. This allows for any manipulations of the source files or build scripts before build. [archive](https://github.com/genotrance/nimarchive/blob/master/nimarchive/archive.nim) has such an example. The build API also includes various compile time helper procs that aid in file manipulation, Cmake shortcuts, library linking, etc. Refer to [build](https://nimterop.github.io/nimterop/build.html) for more details. -__Build__ +#### Build Nimterop currently supports `configure` and `cmake` based building of libraries, with `cmake` taking precedence if a project supports both. Nimterop verifies that the tool selected is available and notifies the user if any issues are found. Bash is required on Windows for `configure` and the binary shipped with Git has been tested. Flags can be specified to these tools via `getHeader()` or directly via the underlying `configure()` and `cmake()` calls. Once the build scripts are ready, `getHeader()` then calls `make()`. At every step, `getHeader()` checks for the presence of created artifacts and does not redo steps that have been successfully completed. -__Linking__ +#### Linking - If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If it is omitted, the dynamic library is returned in `headerLPath`. - `getHeader()` searches for libraries based on the header name by default: @@ -103,7 +103,7 @@ __Linking__ - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. - [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. -__User control__ +#### User control The `-d:xxxYYY` Nim define flags have already been described above and can be specified on the command line or in a nim.cfg file. It is also possible to specify them within the wrapper itself using `setDefines()` if required. Further, all defines, regardless of how they are specified, can be generically checked using `isDefined()`. @@ -132,7 +132,7 @@ cCompile("clib/src/*.c") # Compile in any implementation source files Module documentation for the wrapper API can be found [here](https://nimterop.github.io/nimterop/cimport.html). -__Preprocessing__ +#### Preprocessing In order to leverage the preprocessor, certain projects might need `cDefine()` calls to set `#define` values. Simpler library may have documentation that cover this but larger ones will rely on build tools that discover and set values in a `config.h` which is loaded with `#include`. Projects might also require some `cIncludeDir()` calls to specify paths to directories that contain other headers. This might be within the library or refer to another library. @@ -140,7 +140,7 @@ The wrapper API always runs headers through the C preprocessor before wrapping. By default, the `$CC` environment variable is used for the compiler path. If not found, `toast` defaults to `gcc`. -__Wrapping__ +#### Wrapping The `cImport()` call invokes the `toast` binary with appropriate command line flags including any `cDefine()` and `cInclude()` parameters configured. The output of `toast` is then pulled into the module as Nim code and printed if `cDebug()` is specified. This allows for an end user to simply import the wrapper into their code and access the library API as Nim types and procs. Output is cached to save time on subsequent runs. It is also possible to just redirect the output to a file and import that instead if preferred. @@ -148,13 +148,44 @@ The `recurse` flag can be set to enable the recursion capability which runs thro There may be cases where the wrapper generated by `toast` for certain types or procs is not preferred, or may be skipped or altogether wrong due to limitations or bugs. In these instances, the `cOverride()` macro can be used to define consts, types or procs to use in place of the wrapper generated output. `cImport()` will forward this information to `toast` and the values will be inserted in context in the generated wrapper. This allows wrapper authors to work around tool limitations or to improve the wrapper output - say change `ptr X` to `var X` or to create more Nim friendly types or proc signatures. -Several C libraries also use leading and/or trailing `_` in identifiers and since Nim does not allow this, the `cPlugin()` macro can be used to modify such symbols or `cSkipSymbol()` them altogether. Instead of a full `cPlugin()` section, it might also be preferred to set `flags = "-E_ -F_"` to the `cImport()` call to trim out such characters. These features can also be used to remove common prefixes like `SDL_` to generate a cleaner wrapper. `cPlugin()` is real Nim code though so anything Nim allows is fair game. Note that `cPlugin()` overrides any `-E -F` flags. Also, behind the scenes, `cOverride()` is communicated to `toast` via `cPlugin()`. +Several C libraries also use leading and/or trailing `_` in identifiers and since Nim does not allow this, the `cPlugin()` macro can be used to modify such symbols or `cSkipSymbol()` them altogether. Instead of a full `cPlugin()` section, it might also be preferred to set `flags = "-E_ -F_"` to the `cImport()` call to trim out such characters. These features can also be used to remove common prefixes like `SDL_` to generate a cleaner wrapper. The `--replace | -G` flag can be used for replacements. `cPlugin()` is real Nim code though so anything Nim allows is fair game. Note that `cPlugin()` overrides any `-E -F -G` flags. Also, behind the scenes, `cOverride()` is communicated to `toast` via `cPlugin()`. If the same `cPlugin()` is needed in multiple wrapper files, the code can be moved into a standalone file and be used with the `cPluginPath()` call. -Lastly, `c2nImport()` provides access to calling `c2nim` from the wrapper instead of `toast`. Note that `c2nImport()` does not use any of the above described features like `cPlugin()` and needs to be controlled with the `flags` param. +Lastly, `c2nImport()` provides access to calling `c2nim` from the wrapper instead of `toast`. Note that `c2nImport()` does not use any of the above described features like `cPlugin()` and needs to be controlled with `c2nim` specific flags via the `flags` param. -__Compiling source__ +#### Header vs. Dynlib + +Nim provides some flexibility when it comes to using C/C++ libraries. In order to understand this better, some Nim pragmas need to be introduced. The main one is `{.importc.}` which informs Nim to use a symbol defined in a C library. This applies to both types and procs but how Nim should find the symbol is slightly different for each. + +For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol and to `#include "header.h"` in the generated code. However, types can be mostly recreated in pure Nim so it is also possible to omit both `{.importc.}` and `{.header}` and it will work just fine except with a different name in the generated C code. This allows the user to compile the wrapper without requiring `header.h` to be present. + +For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. + +While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. The user will need to choose based on the library in question. + +Going further, the `{.dynlib: "path/to/libheader.so".}` pragma can be used to inform Nim to load the library at runtime and link the function instead of linking at compile time. This enables creation of a wrapper that does not need the library present at compile time. + +Now that this is understood, a user might want any combination of the above in the wrapper rendered by Nimterop. This can be controlled with various flags to `cImport()` and `toast`. +- By default, generated wrappers will include the `{.header, importc.}` pragmas for types and procs. This can be disabled with the `--noHeader | -H` flag to `toast` or `flags = "-H"` param to `cImport()` which will remove `{.header}` for both and `{.importc.}` for types only. +- By default, generated wrappers will assume that the user will link the library implementation themselves. The `--dynlib | -l` flag to `toast` or `dynlib = "headerLPath"` param to `cImport()` will configure the wrapper to generate `{.dynlib.}` pragmas for procs. + +This results in four cases: +1. Default: `{.header, importc.}` for both types and procs +2. With `--noHeader`, types will be pure Nim and procs will be just `{.importc.}` +3. With `--dynlib`, types will still be `{.header, importc.}` but procs will be `{.dynlib, importc.}` +4. With `--dynlib` and `--noHeader`, types will be pure Nim, procs will be `{.dynlib, importc.}` + +While `ast2` supports all these modes, the legacy backend does not support the third mixed case and will infer `--noHeader` when `--dynlib` is specified (case 4). Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib` for `ast2`. + +More documentation on on these pragmas can be found in the Nim manual: +- [{.importc.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-importc-pragma) +- [{.header.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-header-pragma) +- [{.dynlib.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-dynlib-pragma-for-import) +- [{.passL.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-passl-pragma) +- [{.compile.}](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-pragma) + +#### Compiling the source The job of building and compiling the underlying C library is best left to the build mechanism selected by the library author so using `getHeader()` is recommended. For simpler projects with a few `.c` files though, `cCompile()` should be more than enough. It is not recommended for larger projects which heavily rely on functionality offered by build tools. Recreating reliable logic in Nim can be tedious and one can expect minimal support from that author if their tested build mechanism is not used. @@ -180,13 +211,13 @@ Options: -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" import symbols from library in specified Nim string - -f=, --feature= Features {} flags to enable experimental features - -H, --includeHeader bool false add {.header.} pragma to wrapper + -f=, --feature= Features ast1 flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp --nim= string "nim" use a particular Nim executable - -c, --nocomments bool false exclude top-level comments from output - -o=, --output= string "" file to output content + -c, --noComments bool false exclude top-level comments from output + -H, --noHeader bool false skip {.header.} pragma in wrapper + -o=, --output= string "" file to output content - default: stdout -a, --past bool false print AST output -g, --pgrammar bool false print grammar --pluginSourcePath= string "" nim file to build and load as a plugin @@ -212,7 +243,7 @@ The tree-sitter library is limited though - it may fail on some advanced languag It is debatable whether a syntax highlighting engine like `tree-sitter` is the most reliable method to convert C code into AST. However, it is lightweight, cross-platform with no dependencies and handles error conditions gracefully. It has produced usable wrappers for C libraries though things could get murky when considering C++ but that will be a topic for another day. Nimterop relies heavily on the preprocessor, as discussed next, so having an engine which can run anywhere has been worth the compromise. Only time will tell though. -__Preprocessing__ +### Preprocessing The wrapper API always runs headers through the C preprocessor before wrapping, unlike the command line interface where the `-p | --preprocess` flag is not set by default but *highly* recommended. This is because almost all platform, compiler and package discovery is handled by build tools like `configure` and `cmake` which then use preprocessor `#define` values to tweak what C code is applicable for that platform. While parsing preprocessor macros is possible in tools like `toast`, given how dependent the `#ifdef` branches are on values provided by these and many other build tools, preprocessing seems is best left to them than attempting to self-discover or intercept that information. diff --git a/nimterop.nimble b/nimterop.nimble index 984d0c5..d60224f 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -46,11 +46,13 @@ task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") task test, "Test": + rmFile("tests/timeit.txt") + buildTimeitTask() buildToastTask() execTest "tests/tast2.nim" - execTest "tests/tast2.nim", "-d:HEADER" + execTest "tests/tast2.nim", "-d:NOHEADER" execTest "tests/tnimterop_c.nim" execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 1f1cac6..96116c7 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -2,11 +2,11 @@ import hashes, macros, os, sets, strformat, strutils, tables import regex -import "."/[getters, globals, treesitter/api] +import "."/[getters, globals, treesitter/api, tshelp] proc getHeaderPragma*(gState: State): string = result = - if gState.isIncludeHeader(): + if not gState.noHeader and gState.dynlib.Bl: &", header: {gState.currentHeader}" else: "" @@ -211,7 +211,7 @@ proc parseNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable gState.impShort = gState.currentHeader.replace("header", "imp") gState.sourceFile = fullpath - if gState.isIncludeHeader(): + if not gState.noHeader and gState.dynlib.Bl: gState.constStr &= &"\n {gState.currentHeader} {{.used.}} = \"{fp}\"" root.searchAst(astTable, gState) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6f05a34..1230e06 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -15,7 +15,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS # If not, symbol needs to be skipped - only get here if `name` is blank let # Get cleaned name for symbol, set parent so that cOverride is ignored - name = gState.getIdentifier(origname, kind, parent = "getOverrideOrSkip") + name = gState.getIdentifier(origname, kind, parent = "IgnoreSkipSymbol") override = gState.getOverride(origname, kind) @@ -244,7 +244,8 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: # # If `fname`, use it instead of node.getAtom() for name # If `pragmas`, add as nkPragmaExpr but not for `nskProc` since procs add pragmas elsewhere - # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} when includeHeader + # If `istype` is set, this is a typedef, else struct/union so add {.importc: "struct/union X".} + # when `not noHeader` let atom = node.getAtom() @@ -306,14 +307,14 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: # ) var pragmas = - if gState.isIncludeHeader(): + if not gState.noHeader: # Need to add header and importc if istype and name == origname: - # Need to add impShort since neither struct/union nor name change - pragmas & gState.impShort + # Neither struct/union nor name change + pragmas & "importc" & (gState.impShort & "Hdr") else: # Add header shortcut, additional pragmas added later - pragmas & (gState.impShort & "H") + pragmas & (gState.impShort & "Hdr") else: pragmas @@ -323,7 +324,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: else: ident - if gState.isIncludeHeader(): + if not gState.noHeader: if not istype or name != origname: # Add importc pragma since either struct/union or name changed let @@ -361,18 +362,20 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: prident = block: var prident: PNode + # Add {.importc.} pragma if name != origname: - # Add importc pragma since name changed + # Name changed prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) - if gState.isIncludeHeader(): - # Add header - gState.addPragma(node, prident[1], gState.impShort & "H") - elif gState.dynlib.nBl: - # Add dynlib - gState.addPragma(node, prident[1], "dynlib", gState.getIdent(gState.dynlib)) else: - # Only need impShort since no name change - prident = gState.newPragmaExpr(node, ident, gState.impShort) + prident = gState.newPragmaExpr(node, ident, "importc") + + if gState.dynlib.nBl: + # Add {.dynlib.} + gState.addPragma(node, prident[1], gState.impShort & "Dyn") + elif not gState.noHeader: + # Add {.header.} + gState.addPragma(node, prident[1], gState.impShort & "Hdr") + if pragmas.nBl: gState.addPragma(node, prident[1], pragmas) prident @@ -789,7 +792,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" npexpr = gState.newPragmaExpr(node, typedef[0], pragmas) typedef[0] = npexpr else: - # includeHeader already added impShort in newXIdent() + # `not gState.noHeader` already added pragmas in newXIdent() gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add @@ -1552,23 +1555,20 @@ proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = gState.newPragma(node, "importc", newStrNode(nkStrLit, origname)) else: # {.impnameC.} shortcut - gState.newPragma(node, gState.impShort & "C") + gState.newPragma(node, "importc") # Detect ... and add {.varargs.} pvarargs = plist.getVarargs() - # Need {.convention.} and {.header.} if applicable - if name != origname: - if gState.isIncludeHeader(): - # {.impnameHC.} shortcut - gState.addPragma(node, prident, gState.impShort & "HC") - else: - # {.convention.} - gState.addPragma(node, prident, gState.convention) + # Need {.convention.} + gState.addPragma(node, prident, gState.convention) - if gState.dynlib.nBl: - # {.dynlib.} for DLLs - gState.addPragma(node, prident, "dynlib", gState.getIdent(gState.dynlib)) + if gState.dynlib.nBl: + # Add {.dynlib.} + gState.addPragma(node, prident, gState.impShort & "Dyn") + elif not gState.noHeader: + # Add {.header.} + gState.addPragma(node, prident, gState.impShort & "Hdr") if pvarargs: # Add {.varargs.} for ... @@ -1637,10 +1637,10 @@ proc addDef(gState: State, node: TSNode) = commentNodes = gState.getCommentNodes(node) if node[start+1].getName() == "function_declarator": - if gState.isIncludeHeader(): + if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: - gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % + gecho &"\n# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" % gState.getNodeVal(node[start+1].getAtom()) proc processNode(gState: State, node: TSNode): bool = @@ -1707,50 +1707,35 @@ proc searchTree(gState: State, root: TSNode) = if node == root: break +### +# gState.addPragma(root, impPragma, "importc") +# gState.addPragma(root, impConvPragma, gState.convention) + proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # Create shortcut pragmas to reduce clutter var hdrPragma: PNode - hdrConvPragma: PNode - impPragma = newNode(nkPragma) - impConvPragma = newNode(nkPragma) + dynPragma: PNode - # {.pragma: impname, importc.} - gState.addPragma(root, impPragma, "pragma", gState.getIdent(gState.impShort)) - gState.addPragma(root, impPragma, "importc") - - if gState.isIncludeHeader(): + if not gState.noHeader: # Path to header const gState.constSection.add gState.newConstDef( root, fname = gState.currentHeader, fval = '"' & fullpath & '"') - # {.pragma: impnameH, header: "xxx".} for types when name != origname - hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "H")) + # {.pragma: impnameHdr, header: "xxx".} + hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) gState.addPragma(root, hdrPragma, "header", gState.getIdent(gState.currentHeader)) - # Add {.impnameH.} to {.impname.} - gState.addPragma(root, impPragma, gState.impShort & "H") - - # {.pragma: impnameHC, impnameH, convention.} for procs when name != origname - hdrConvPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "HC")) - gState.addPragma(root, hdrConvPragma, gState.impShort & "H") - gState.addPragma(root, hdrConvPragma, gState.convention) - - # {.pragma: impnameC, impname, convention.} for procs - gState.addPragma(root, impConvPragma, "pragma", gState.getIdent(gState.impShort & "C")) - gState.addPragma(root, impConvPragma, gState.impShort) - gState.addPragma(root, impConvPragma, gState.convention) - if gState.dynlib.nBl: - # {.dynlib.} for DLLs - gState.addPragma(root, impConvPragma, "dynlib", gState.getIdent(gState.dynlib)) + # {.pragma: impnameDyn, dynlib: libname.} + dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) + gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) - # Add all pragma shortcuts to output + # Add pragma shortcuts to output if not hdrPragma.isNil: gState.pragmaSection.add hdrPragma - gState.pragmaSection.add hdrConvPragma - gState.pragmaSection.add impPragma - gState.pragmaSection.add impConvPragma + if not dynPragma.isNil: + gState.pragmaSection.add dynPragma proc printNimHeader*(gState: State) = # Top level output with context info diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim index 18915b0..e577717 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/comphelp.nim @@ -1,6 +1,8 @@ -import compiler/[ast, lineinfos, msgs, options, parser, renderer] +import macros, strutils -import "."/[globals, getters] +import compiler/[ast, idents, lineinfos, msgs, options, parser, pathutils, renderer] + +import "."/[globals, getters, treesitter/api, tshelp] proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = # Raise exception in parseString() instead of exiting for errors @@ -17,6 +19,84 @@ proc parseString*(gState: State, str: string): PNode = except: decho getCurrentExceptionMsg() +proc printTree*(gState: State, pnode: PNode, offset = ""): string = + if not pnode.isNil and gState.debug and pnode.kind != nkNone: + result &= "\n# " & offset & $pnode.kind & "(" + case pnode.kind + of nkCharLit: + result &= ($pnode.intVal.char).escape & ")" + of nkIntLit..nkUInt64Lit: + result &= $pnode.intVal & ")" + of nkFloatLit..nkFloat128Lit: + result &= $pnode.floatVal & ")" + of nkStrLit..nkTripleStrLit: + result &= pnode.strVal.escape & ")" + of nkSym: + result &= $pnode.sym & ")" + of nkIdent: + result &= "\"" & $pnode.ident.s & "\")" + else: + if pnode.sons.len != 0: + for i in 0 ..< pnode.sons.len: + result &= gState.printTree(pnode.sons[i], offset & " ") + if i != pnode.sons.len - 1: + result &= "," + result &= "\n# " & offset & ")" + else: + result &= ")" + if offset.len == 0: + result &= "\n" + +proc printDebug*(gState: State, pnode: PNode) = + if gState.debug and pnode.kind != nkNone: + gecho ("Output => " & $pnode).getCommented() + gecho gState.printTree(pnode) + +proc getDefaultLineInfo*(gState: State): TLineInfo = + result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, 0, 0) + +proc getLineInfo*(gState: State, node: TSNode): TLineInfo = + # Get Nim equivalent line:col info from node + let + (line, col) = gState.getLineCol(node) + + result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, line, col) + +proc getIdent*(gState: State, name: string, info: TLineInfo, exported = true): PNode = + if name.nBl: + # Get ident PNode for name + info + let + exp = getIdent(gState.identCache, "*") + ident = getIdent(gState.identCache, name) + + if exported: + result = newNode(nkPostfix) + result.add newIdentNode(exp, info) + result.add newIdentNode(ident, info) + else: + result = newIdentNode(ident, info) + +proc getIdent*(gState: State, name: string): PNode = + gState.getIdent(name, gState.getDefaultLineInfo(), exported = false) + +proc getIdentName*(node: PNode): string = + if not node.isNil: + for i in 0 ..< node.len: + if node[i].kind == nkIdent and $node[i] != "*": + result = $node[i] + if result.Bl and node.len > 0: + result = node[0].getIdentName() + +proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): + tuple[name, origname: string, info: TLineInfo] = + # Shortcut to get identifier name and info (node value and line:col) + result.origname = gState.getNodeVal(node) + result.name = gState.getIdentifier(result.origname, kind, parent) + if result.name.nBl: + if kind == nskType: + result.name = result.name.getType() + result.info = gState.getLineInfo(node) + proc getPtrType*(str: string): string = result = case str: of "cchar": diff --git a/nimterop/docs.nim b/nimterop/docs.nim index d5bd50c..6c2bbfd 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -74,13 +74,13 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath for file in files: echo execAction(&"{nim} doc {defStr} {nimArgs} -o:{path} --project --index:on {baseDir & file}") - echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}") + echo execAction(&"{nim} buildIndex {nimArgs} -o:{path}/theindex.html {path}") when declared(getNimRootDir): #[ this enables doc search, works at least locally with: cd {path} && python -m SimpleHTTPServer 9009 ]# - echo execAction(&"{nim} js -o:{path}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim") + echo execAction(&"{nim} js {nimArgs} -o:{path}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim") for i in 0 .. paramCount(): if paramStr(i) == "--publish": diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 10c3b5e..12d08eb 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,9 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] - -import "."/[build, globals, plugin, treesitter/api] +import "."/[build, globals, plugin] const gReserved = """ addr and as asm @@ -135,7 +133,12 @@ proc checkIdentifier(name, kind, parent, origName: string) = doAssert (not name.contains("__")): errmsg & " consecutive underscores '_'" - if parent.nBl: + # Cannot blank out symbols which are fields or params + # + # `IgnoreSkipSymbol` is used to `getIdentifier()` even if symbol is in `symOverride` list + # so that any prefix/suffix/replace or `onSymbol()` processing can occur. This is only used + # for `cOverride()` since it also depends on `symOverride`. + if parent.nBl and parent != "IgnoreSkipSymbol": doAssert name.nBl, &"Blank identifier, originally '{parentStr}{origName}' ({kind}), cannot be empty" proc getIdentifier*(gState: State, name: string, kind: NimSymKind, parent=""): string = @@ -211,7 +214,7 @@ proc getOverride*(gState: State, name: string, kind: NimSymKind): string = if gState.onSymbolOverride != nil: var - nname = gState.getIdentifier(name, kind, "Override") + nname = gState.getIdentifier(name, kind, "IgnoreSkipSymbol") sym = Symbol(name: nname, kind: kind) if nname.nBl: gState.onSymbolOverride(sym) @@ -236,327 +239,12 @@ proc getKeyword*(kind: NimSymKind): string = # cOverride procs already include `proc` keyword result = ($kind).replace("nsk", "").toLowerAscii() -# TSNode shortcuts - -proc isNil*(node: TSNode): bool = - node.tsNodeIsNull() - -proc len*(node: TSNode): int = - if not node.isNil: - result = node.tsNodeNamedChildCount().int - -proc `[]`*(node: TSNode, i: SomeInteger): TSNode = - if i < type(i)(node.len()): - result = node.tsNodeNamedChild(i.uint32) - -proc getName*(node: TSNode): string {.inline.} = - if not node.isNil: - return $node.tsNodeType() - -proc getNodeVal*(code: var string, node: TSNode): string = - if not node.isNil: - return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() - -proc getNodeVal*(gState: State, node: TSNode): string = - gState.code.getNodeVal(node) - -proc getAtom*(node: TSNode): TSNode = - if not node.isNil: - # Get child node which is topmost atom - if node.getName() in gAtoms: - return node - elif node.len != 0: - if node[0].getName() == "type_qualifier": - # Skip const, volatile - if node.len > 1: - return node[1].getAtom() - else: - return - else: - return node[0].getAtom() - -proc getStartAtom*(node: TSNode): int = - if not node.isNil: - # Skip const, volatile and other type qualifiers - for i in 0 .. node.len - 1: - if node[i].getAtom().getName() notin gAtoms: - result += 1 - else: - break - -proc getXCount*(node: TSNode, ntype: string, reverse = false): int = - if not node.isNil: - # Get number of ntype nodes nested in tree - var - cnode = node - while ntype in cnode.getName(): - result += 1 - if reverse: - cnode = cnode.tsNodeParent() - else: - if cnode.len != 0: - if cnode[0].getName() == "type_qualifier": - # Skip const, volatile - if cnode.len > 1: - cnode = cnode[1] - else: - break - else: - cnode = cnode[0] - else: - break - -proc getPtrCount*(node: TSNode, reverse = false): int = - node.getXCount("pointer_declarator") - -proc getArrayCount*(node: TSNode, reverse = false): int = - node.getXCount("array_declarator") - -proc getDeclarator*(node: TSNode): TSNode = - if not node.isNil: - # Return if child is a function or array declarator - if node.getName() in ["function_declarator", "array_declarator"]: - return node - elif node.len != 0: - return node[0].getDeclarator() - -proc getVarargs*(node: TSNode): bool = - # Detect ... and add {.varargs.} - # - # `node` is the param list - # - # ... is an unnamed node, second last node and ) is last node - let - nlen = node.tsNodeChildCount() - if nlen > 1.uint32: - let - nval = node.tsNodeChild(nlen - 2.uint32).getName() - if nval == "...": - result = true - -proc firstChildInTree*(node: TSNode, ntype: string): TSNode = - # Search for node type in tree - first children - var - cnode = node - while not cnode.isNil: - if cnode.getName() == ntype: - return cnode - cnode = cnode[0] - -proc anyChildInTree*(node: TSNode, ntype: string): TSNode = - # Search for node type anywhere in tree - depth first - var - cnode = node - while not cnode.isNil: - if cnode.getName() == ntype: - return cnode - for i in 0 ..< cnode.len: - let - ccnode = cnode[i].anyChildInTree(ntype) - if not ccnode.isNil: - return ccnode - if cnode != node: - cnode = cnode.tsNodeNextNamedSibling() - else: - break - -proc mostNestedChildInTree*(node: TSNode): TSNode = - # Search for the most nested child of node's type in tree - var - cnode = node - ntype = cnode.getName() - while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype: - cnode = cnode[0] - result = cnode - -proc inChildren*(node: TSNode, ntype: string): bool = - # Search for node type in immediate children - result = false - for i in 0 ..< node.len: - if (node[i]).getName() == ntype: - result = true - break - -proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = - # Get line number and column info for node - let - point = node.tsNodeStartPoint() - result.line = point.row.int + 1 - result.col = point.column.int + 1 - -proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = - getLineCol(gState.code, node) - -proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] = - # Get line number and column info for node - let - point = node.tsNodeEndPoint() - result.line = point.row.int + 1 - result.col = point.column.int + 1 - -proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] = - getEndLineCol(gState.code, node) - -proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = - for i in 0 ..< node.len: - if node.getName() != "comment": - result += 1 - -proc getPxName*(node: TSNode, offset: int): string = - # Get the xth (grand)parent of the node - var - np = node - count = 0 - - while not np.isNil and count < offset: - np = np.tsNodeParent() - count += 1 - - if count == offset and not np.isNil: - return np.getName() - -proc printLisp*(code: var string, root: TSNode): string = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.isNil and depth > -1: - result &= spaces(depth) - let - (line, col) = code.getLineCol(node) - result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" - let - val = code.getNodeVal(node) - if "\n" notin val and " " notin val: - result &= &" \"{val}\"" - else: - break - - if node.len() != 0: - result &= "\n" - nextnode = node[0] - depth += 1 - else: - result &= ")\n" - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.isNil: - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - result &= spaces(depth) & ")\n" - if node == root: - break - if not node.tsNodeNextNamedSibling().isNil: - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break - -proc printLisp*(gState: State, root: TSNode): string = - printLisp(gState.code, root) - -proc getCommented*(str: string): string = - "\n# " & str.strip().replace("\n", "\n# ") - -proc printTree*(gState: State, pnode: PNode, offset = ""): string = - if not pnode.isNil and gState.debug and pnode.kind != nkNone: - result &= "\n# " & offset & $pnode.kind & "(" - case pnode.kind - of nkCharLit: - result &= ($pnode.intVal.char).escape & ")" - of nkIntLit..nkUInt64Lit: - result &= $pnode.intVal & ")" - of nkFloatLit..nkFloat128Lit: - result &= $pnode.floatVal & ")" - of nkStrLit..nkTripleStrLit: - result &= pnode.strVal.escape & ")" - of nkSym: - result &= $pnode.sym & ")" - of nkIdent: - result &= "\"" & $pnode.ident.s & "\")" - else: - if pnode.sons.len != 0: - for i in 0 ..< pnode.sons.len: - result &= gState.printTree(pnode.sons[i], offset & " ") - if i != pnode.sons.len - 1: - result &= "," - result &= "\n# " & offset & ")" - else: - result &= ")" - if offset.len == 0: - result &= "\n" - -proc printDebug*(gState: State, node: TSNode) = - if gState.debug: - gecho ("Input => " & gState.getNodeVal(node)).getCommented() - gecho gState.printLisp(node).getCommented() - -proc printDebug*(gState: State, pnode: PNode) = - if gState.debug and pnode.kind != nkNone: - gecho ("Output => " & $pnode).getCommented() - gecho gState.printTree(pnode) - -# Compiler shortcuts - -proc getDefaultLineInfo*(gState: State): TLineInfo = - result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, 0, 0) - -proc getLineInfo*(gState: State, node: TSNode): TLineInfo = - # Get Nim equivalent line:col info from node - let - (line, col) = gState.getLineCol(node) - - result = newLineInfo(gState.config, gState.sourceFile.AbsoluteFile, line, col) - -proc getIdent*(gState: State, name: string, info: TLineInfo, exported = true): PNode = - if name.nBl: - # Get ident PNode for name + info - let - exp = getIdent(gState.identCache, "*") - ident = getIdent(gState.identCache, name) - - if exported: - result = newNode(nkPostfix) - result.add newIdentNode(exp, info) - result.add newIdentNode(ident, info) - else: - result = newIdentNode(ident, info) - -proc getIdent*(gState: State, name: string): PNode = - gState.getIdent(name, gState.getDefaultLineInfo(), exported = false) - -proc getIdentName*(node: PNode): string = - if not node.isNil: - for i in 0 ..< node.len: - if node[i].kind == nkIdent and $node[i] != "*": - result = $node[i] - if result.Bl and node.len > 0: - result = node[0].getIdentName() - -proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): - tuple[name, origname: string, info: TLineInfo] = - # Shortcut to get identifier name and info (node value and line:col) - result.origname = gState.getNodeVal(node) - result.name = gState.getIdentifier(result.origname, kind, parent) - if result.name.nBl: - if kind == nskType: - result.name = result.name.getType() - result.info = gState.getLineInfo(node) - proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) proc getPreprocessor*(gState: State, fullpath: string): string = var - cmts = if gState.nocomments: "" else: "-CC" + cmts = if gState.noComments: "" else: "-CC" cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ rdata: seq[string] = @[] @@ -644,118 +332,6 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] -proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = - ## Generate a comment from a set of comment nodes. Comment is guaranteed - ## to be able to be rendered using nim doc - if commentNodes.len > 0: - result = "::" - for commentNode in commentNodes: - result &= "\n " & gState.getNodeVal(commentNode).strip() - - result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") - result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() - -proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = - ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` - ## nodes before and after the current node - ## - ## Priority is (closest line number) > comment before > comment after. - ## This priority might need to be changed based on the project, but - ## for now it is good enough - - # Skip this if we don't want comments - if gState.nocomments: - return - - let (line, _) = gState.getLineCol(node) - - # Keep track of both directions from a node - var - prevSibling = node.tsNodePrevNamedSibling() - nextSibling = node.tsNodeNextNamedSibling() - nilNode: TSNode - - var - i = 0 - prevSiblingDistance, nextSiblingDistance: int = int.high - lowestDistance: int - commentsFound = false - - while not commentsFound and i < maxSearch: - # Distance from the current node will tell us approximately if the - # comment belongs to the node. The closer it is in terms of line - # numbers, the more we can be sure it's the comment we want - if not prevSibling.isNil: - if prevSibling.getName() == "comment": - prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line) - else: - prevSiblingDistance = int.high - if not nextSibling.isNil: - if nextSibling.getName() == "comment": - nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) - else: - nextSiblingDistance = int.high - - lowestDistance = min(prevSiblingDistance, nextSiblingDistance) - - if prevSiblingDistance > maxSearch: - # If the line is out of range, skip searching - prevSibling = nilNode # Can't do `= nil` - - if nextSiblingDistance > maxSearch: - # If the line is out of range, skip searching - nextSibling = nilNode - - # Search above the current line for comments. When one is found - # keep going to retrieve successive comments for cases with multiple - # `//` style comments - while ( - not prevSibling.isNil and - prevSibling.getName() == "comment" and - prevSiblingDistance == lowestDistance - ): - # Put the previous nodes in reverse order so the comments - # make logical sense - result.insert(prevSibling, 0) - prevSibling = prevSibling.tsNodePrevNamedSibling() - commentsFound = true - - # If we've already found comments above the current line, quit - if commentsFound: - break - - # Search below or at the current line for comments. When one is found - # keep going to retrieve successive comments for cases with multiple - # `//` style comments - while ( - not nextSibling.isNil and - nextSibling.getName() == "comment" and - nextSiblingDistance == lowestDistance - ): - result.add(nextSibling) - nextSibling = nextSibling.tsNodeNextNamedSibling() - commentsFound = true - - if commentsFound: - break - - # Go to next sibling pair - if not prevSibling.isNil: - prevSibling = prevSibling.tsNodePrevNamedSibling() - if not nextSibling.isNil: - nextSibling = nextSibling.tsNodeNextNamedSibling() - - i += 1 - -proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - let - name = $node.tsNodeNamedChild(i).tsNodeType() - - if name != "comment": - result.add(name) - proc getRegexForAstChildren*(ast: ref Ast): string = result = "^" for i in 0 .. ast.children.len-1: @@ -837,20 +413,15 @@ proc getNimExpression*(gState: State, expr: string, name = ""): string = ("<<", " shl "), (">>", " shr ") ]) -proc getSplitComma*(joined: seq[string]): seq[string] = - for i in joined: - result = result.concat(i.split(",")) - -template isIncludeHeader*(gState: State): bool = - gState.dynlib.Bl and gState.includeHeader - proc getComments*(gState: State, strip = false): string = - if not gState.nocomments and gState.commentStr.nBl: + if not gState.noComments and gState.commentStr.nBl: result = "\n" & gState.commentStr if strip: result = result.replace("\n ", "\n") gState.commentStr = "" +# Plugin related + proc dll*(path: string): string = let (dir, name, _) = path.splitFile() @@ -888,6 +459,12 @@ proc loadPlugin*(gState: State, sourcePath: string) = gState.onSymbolOverrideFinal = cast[OnSymbolOverrideFinal](lib.symAddr("onSymbolOverrideFinal")) +# Misc toast helpers + +proc getSplitComma*(joined: seq[string]): seq[string] = + for i in joined: + result = result.concat(i.split(",")) + proc expandSymlinkAbs*(path: string): string = try: result = path.expandFilename().normalizedPath() diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 21214f3..05f507c 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -43,39 +43,6 @@ type zeroOrOne # ? orWithNext # ! - PragmaMode* = enum - # includeHeader = true, dynlib = false - # - # Types = {.header, .importc.} - # Static inline = {.header, .importc.} - # Procs = {.header, .importc.} - # Lib = {.compile.} or {.passL.} - useHeader - - # includeHeader = true, dynlib = true - # - # Types = {.header, .importc.} - # Static inline = {.header, .importc.} - # Procs = {.dynlib, importc.} - # Libs = loaded by dynlib - useHeaderDynlib - - # includeHeader = false, dynlib = false - # - # Types = {.bycopy.} - # Static inline = not available - # Procs = {.importc.} - # Lib = {.compile.} or {.passL.} - noHeader - - # includeHeader = false, dynlib = true - # - # Types = {.bycopy.} - # Static inline = not available - # Procs = {.dynlib, importc.} - # Libs = loaded by dynlib - noHeaderDynlib - Ast* = object name*: string kind*: Kind @@ -95,10 +62,10 @@ type dynlib*: string # `cImport(dynlib)` or `--dynlib | -l` to specify variable containing library name feature*: seq[Feature] # `--feature | -f` feature flags enabled includeDirs*: seq[string] # Paths added by `cIncludeDir()` and `--includeDirs | -I` for C/C++ preprocessor/compiler - includeHeader*: bool # `--includeHeader | -H` to include {.header.} pragma to wrapper mode*: string # `cImport(mode)` or `--mode | -m` to override detected compiler mode - c or cpp nim*: string # `--nim` to specify full path to Nim compiler - nocomments*: bool # `--nocomments | -c` to disable rendering comments in wrappers + noComments*: bool # `--noComments | -c` to disable rendering comments in wrappers + noHeader*: bool # `--noHeader | -H` to skip {.header.} pragma in wrapper past*: bool # `--past | -a` to print tree-sitter AST of code pluginSourcePath*: string # `--pluginSourcePath` specified path to plugin file to compile and load pnim*: bool # `--pnim | -n` to render Nim wrapper for header diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index eaa337a..e061a70 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -2,7 +2,7 @@ import macros, strformat, strutils, tables import regex -import "."/[ast, getters, globals, lisp, treesitter/api] +import "."/[ast, getters, globals, lisp, treesitter/api, tshelp] type Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] @@ -180,7 +180,7 @@ proc initGrammar(): Grammar = var i = 0 - typ = gState.getIdentifier(gState.data[i].val, nskType, "Parent").getType() + typ = gState.getIdentifier(gState.data[i].val, nskType, "IgnoreSkipSymbol").getType() name = "" nname = "" tptr = "" @@ -202,7 +202,7 @@ proc initGrammar(): Grammar = nname = gState.getIdentifier(name, nskType) i += 1 - if gState.isIncludeHeader(): + if not gState.noHeader and gState.dynlib.Bl: pragmas.add gState.getImportC(name, nname) let @@ -318,7 +318,7 @@ proc initGrammar(): Grammar = else: var pragmas: seq[string] = @[] - if gState.isIncludeHeader(): + if not gState.noHeader and gState.dynlib.Bl: pragmas.add gState.getImportC(prefix & name, nname) pragmas.add "bycopy" if union.nBl: diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 78708f5..d74838a 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -34,11 +34,11 @@ proc main( defines: seq[string] = @[], dynlib: string = "", feature: seq[Feature] = @[Feature.ast1], - includeHeader = false, includeDirs: seq[string] = @[], mode = "", nim: string = "nim", - nocomments = false, + noComments = false, + noHeader = false, output = "", past = false, pgrammar = false, @@ -61,11 +61,11 @@ proc main( defines: defines, dynlib: dynlib, feature: feature, - includeHeader: includeHeader, includeDirs: includeDirs, mode: mode, nim: nim, - nocomments: nocomments, + noComments: noComments, + noHeader: noHeader, past: past, pluginSourcePath: pluginSourcePath, pnim: pnim, @@ -77,10 +77,6 @@ proc main( symOverride: symOverride ) - # Fail if both includeHeader and dynlib - doAssert not (includeHeader == true and dynlib.nBl), - "`includeHeader` and `dynlib` cannot be used simultaneously" - # Set gDebug in build.nim build.gDebug = debug @@ -196,16 +192,16 @@ when isMainModule: import cligen dispatch(main, help = { "check": "check generated wrapper with compiler", - "convention": "calling convention for wrapped procs - default: cdecl", + "convention": "calling convention for wrapped procs", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", "dynlib": "import symbols from library in specified Nim string", "feature": "flags to enable experimental features", - "includeHeader": "add {.header.} pragma to wrapper", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", - "nim": "use a particular Nim executable - default: $PATH/nim", - "nocomments": "exclude top-level comments from output", + "nim": "use a particular Nim executable", + "noComments": "exclude top-level comments from output", + "noHeader": "skip {.header.} pragma in wrapper", "output": "file to output content - default: stdout", "past": "print AST output", "pgrammar": "print grammar", @@ -226,9 +222,9 @@ when isMainModule: "defines": 'D', "dynlib": 'l', "feature": 'f', - "includeHeader": 'H', "includeDirs": 'I', - "nocomments": 'c', + "noComments": 'c', + "noHeader": 'H', "output": 'o', "past": 'a', "pgrammar": 'g', diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 109321c..762cf7c 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,4 +1,9 @@ -import "."/treesitter/[c, cpp] +import sets, strformat, strutils + +import regex + +import "."/[getters, globals] +import "."/treesitter/[api, c, cpp] template withCodeAst*(code: string, mode: string, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code @@ -25,4 +30,349 @@ template withCodeAst*(code: string, mode: string, body: untyped): untyped = body defer: - tree.tsTreeDelete() \ No newline at end of file + tree.tsTreeDelete() + +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +proc isNil*(node: TSNode): bool = + node.tsNodeIsNull() + +proc len*(node: TSNode): int = + if not node.isNil: + result = node.tsNodeNamedChildCount().int + +proc `[]`*(node: TSNode, i: SomeInteger): TSNode = + if i < type(i)(node.len()): + result = node.tsNodeNamedChild(i.uint32) + +proc getName*(node: TSNode): string {.inline.} = + if not node.isNil: + return $node.tsNodeType() + +proc getNodeVal*(code: var string, node: TSNode): string = + if not node.isNil: + return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + +proc getNodeVal*(gState: State, node: TSNode): string = + gState.code.getNodeVal(node) + +proc getAtom*(node: TSNode): TSNode = + if not node.isNil: + # Get child node which is topmost atom + if node.getName() in gAtoms: + return node + elif node.len != 0: + if node[0].getName() == "type_qualifier": + # Skip const, volatile + if node.len > 1: + return node[1].getAtom() + else: + return + else: + return node[0].getAtom() + +proc getStartAtom*(node: TSNode): int = + if not node.isNil: + # Skip const, volatile and other type qualifiers + for i in 0 .. node.len - 1: + if node[i].getAtom().getName() notin gAtoms: + result += 1 + else: + break + +proc getXCount*(node: TSNode, ntype: string, reverse = false): int = + if not node.isNil: + # Get number of ntype nodes nested in tree + var + cnode = node + while ntype in cnode.getName(): + result += 1 + if reverse: + cnode = cnode.tsNodeParent() + else: + if cnode.len != 0: + if cnode[0].getName() == "type_qualifier": + # Skip const, volatile + if cnode.len > 1: + cnode = cnode[1] + else: + break + else: + cnode = cnode[0] + else: + break + +proc getPtrCount*(node: TSNode, reverse = false): int = + node.getXCount("pointer_declarator") + +proc getArrayCount*(node: TSNode, reverse = false): int = + node.getXCount("array_declarator") + +proc getDeclarator*(node: TSNode): TSNode = + if not node.isNil: + # Return if child is a function or array declarator + if node.getName() in ["function_declarator", "array_declarator"]: + return node + elif node.len != 0: + return node[0].getDeclarator() + +proc getVarargs*(node: TSNode): bool = + # Detect ... and add {.varargs.} + # + # `node` is the param list + # + # ... is an unnamed node, second last node and ) is last node + let + nlen = node.tsNodeChildCount() + if nlen > 1.uint32: + let + nval = node.tsNodeChild(nlen - 2.uint32).getName() + if nval == "...": + result = true + +proc firstChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type in tree - first children + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + cnode = cnode[0] + +proc anyChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type anywhere in tree - depth first + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + for i in 0 ..< cnode.len: + let + ccnode = cnode[i].anyChildInTree(ntype) + if not ccnode.isNil: + return ccnode + if cnode != node: + cnode = cnode.tsNodeNextNamedSibling() + else: + break + +proc mostNestedChildInTree*(node: TSNode): TSNode = + # Search for the most nested child of node's type in tree + var + cnode = node + ntype = cnode.getName() + while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype: + cnode = cnode[0] + result = cnode + +proc inChildren*(node: TSNode, ntype: string): bool = + # Search for node type in immediate children + result = false + for i in 0 ..< node.len: + if (node[i]).getName() == ntype: + result = true + break + +proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node + let + point = node.tsNodeStartPoint() + result.line = point.row.int + 1 + result.col = point.column.int + 1 + +proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getLineCol(gState.code, node) + +proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node + let + point = node.tsNodeEndPoint() + result.line = point.row.int + 1 + result.col = point.column.int + 1 + +proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getEndLineCol(gState.code, node) + +proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = + for i in 0 ..< node.len: + if node.getName() != "comment": + result += 1 + +proc getPxName*(node: TSNode, offset: int): string = + # Get the xth (grand)parent of the node + var + np = node + count = 0 + + while not np.isNil and count < offset: + np = np.tsNodeParent() + count += 1 + + if count == offset and not np.isNil: + return np.getName() + +proc printLisp*(code: var string, root: TSNode): string = + var + node = root + nextnode: TSNode + depth = 0 + + while true: + if not node.isNil and depth > -1: + result &= spaces(depth) + let + (line, col) = code.getLineCol(node) + result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + let + val = code.getNodeVal(node) + if "\n" notin val and " " notin val: + result &= &" \"{val}\"" + else: + break + + if node.len() != 0: + result &= "\n" + nextnode = node[0] + depth += 1 + else: + result &= ")\n" + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.isNil: + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + result &= spaces(depth) & ")\n" + if node == root: + break + if not node.tsNodeNextNamedSibling().isNil: + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc printLisp*(gState: State, root: TSNode): string = + printLisp(gState.code, root) + +proc printDebug*(gState: State, node: TSNode) = + if gState.debug: + gecho ("Input => " & gState.getNodeVal(node)).getCommented() + gecho gState.printLisp(node).getCommented() + +proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = + ## Generate a comment from a set of comment nodes. Comment is guaranteed + ## to be able to be rendered using nim doc + if commentNodes.len > 0: + result = "::" + for commentNode in commentNodes: + result &= "\n " & gState.getNodeVal(commentNode).strip() + + result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") + result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() + +proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = + ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` + ## nodes before and after the current node + ## + ## Priority is (closest line number) > comment before > comment after. + ## This priority might need to be changed based on the project, but + ## for now it is good enough + + # Skip this if we don't want comments + if gState.noComments: + return + + let (line, _) = gState.getLineCol(node) + + # Keep track of both directions from a node + var + prevSibling = node.tsNodePrevNamedSibling() + nextSibling = node.tsNodeNextNamedSibling() + nilNode: TSNode + + var + i = 0 + prevSiblingDistance, nextSiblingDistance: int = int.high + lowestDistance: int + commentsFound = false + + while not commentsFound and i < maxSearch: + # Distance from the current node will tell us approximately if the + # comment belongs to the node. The closer it is in terms of line + # numbers, the more we can be sure it's the comment we want + if not prevSibling.isNil: + if prevSibling.getName() == "comment": + prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line) + else: + prevSiblingDistance = int.high + if not nextSibling.isNil: + if nextSibling.getName() == "comment": + nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) + else: + nextSiblingDistance = int.high + + lowestDistance = min(prevSiblingDistance, nextSiblingDistance) + + if prevSiblingDistance > maxSearch: + # If the line is out of range, skip searching + prevSibling = nilNode # Can't do `= nil` + + if nextSiblingDistance > maxSearch: + # If the line is out of range, skip searching + nextSibling = nilNode + + # Search above the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments + while ( + not prevSibling.isNil and + prevSibling.getName() == "comment" and + prevSiblingDistance == lowestDistance + ): + # Put the previous nodes in reverse order so the comments + # make logical sense + result.insert(prevSibling, 0) + prevSibling = prevSibling.tsNodePrevNamedSibling() + commentsFound = true + + # If we've already found comments above the current line, quit + if commentsFound: + break + + # Search below or at the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments + while ( + not nextSibling.isNil and + nextSibling.getName() == "comment" and + nextSiblingDistance == lowestDistance + ): + result.add(nextSibling) + nextSibling = nextSibling.tsNodeNextNamedSibling() + commentsFound = true + + if commentsFound: + break + + # Go to next sibling pair + if not prevSibling.isNil: + prevSibling = prevSibling.tsNodePrevNamedSibling() + if not nextSibling.isNil: + nextSibling = nextSibling.tsNodeNextNamedSibling() + + i += 1 + +proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = + if node.tsNodeNamedChildCount() != 0: + for i in 0 .. node.tsNodeNamedChildCount()-1: + let + name = $node.tsNodeNamedChild(i).tsNodeType() + + if name != "comment": + result.add(name) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 0b1fca2..878a3a2 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -247,7 +247,7 @@ static inline int sitest1(int f1) { // DUPLICATES -#ifndef HEADER +#ifdef NOHEADER #define A 1 #define B 1.0 diff --git a/tests/tast2.nim b/tests/tast2.nim index 6cdd606..bd040e2 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -11,17 +11,17 @@ static: const path = currentSourcePath.parentDir() / "include" / "tast2.h" -when defined(HEADER): - cDefine("HEADER") +when defined(NOHEADER): + cDefine("NOHEADER") const flags = " -H" - pHeader = @["header:" & path.replace("\\", "/")] - pHeaderImp = @["importc"] & pHeader + pHeader: seq[string] = @[] + pHeaderImp: seq[string] = @[] else: const flags = "" - pHeader: seq[string] = @[] - pHeaderImp: seq[string] = @[] + pHeader = @["header:" & path.replace("\\", "/")] + pHeaderImp = @["importc"] & pHeader const pHeaderImpBy = @["bycopy"] & pHeaderImp @@ -68,7 +68,7 @@ macro checkPragmas(t: typed, pragmas: static[seq[string]], istype: static[bool] ast = t.getImpl() prag = ast.getPragmas() exprag = pragmas.toHashSet() - when defined(HEADER): + when not defined(NOHEADER): if not istype: if "union" in exprag: exprag.incl "importc:union " & $t @@ -169,7 +169,7 @@ a1.f1 = 2 assert A2 is object testFields(A2) checkPragmas(A2, pHeaderInc, istype = false) -when not defined(HEADER): +when defined(NOHEADER): # typedef struct X; is invalid var a2: A2 @@ -475,6 +475,6 @@ assert nested is object testFields(nested, "f1|f2|f3|f4|f5|f6|f7|f8!NT1|Type_tast2h1|NT3|Type_tast2h3|NU2|Union_tast2h1|NE1|Enum_tast2h2") checkPragmas(nested, pHeaderImpBy) -when defined(HEADER): +when not defined(NOHEADER): assert sitest1(5) == 10 assert sitest1(10) == 20 diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 51530c1..4a3badb 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -28,6 +28,8 @@ cPlugin: proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = sym.name = sym.name.replace("pcre_", "") + if sym.name.startsWith("pcre16_") or sym.name.startsWith("pcre32_"): + sym.name = "" const FLAGS {.strdefine.} = "" cImport(pcreH, dynlib="dynpcre", flags="--mode=c " & FLAGS) From e2125768f4a1e03ee7acd11303a293b64e235b97 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 9 May 2020 17:40:49 -0500 Subject: [PATCH 133/255] Fix #207 - better error message --- nimterop/getters.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 12d08eb..f1d1866 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -127,11 +127,12 @@ proc checkIdentifier(name, kind, parent, origName: string) = if name.nBl: let origStr = if name != origName: &", originally '{origName}' before 'cPlugin:onSymbol()', still" else: "" - errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} contains" + errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} contains $1 " & + "which Nim does not allow. Use toast flag '$2' or 'cPlugin()' to modify." - doAssert name[0] != '_' and name[^1] != '_', errmsg & " leading/trailing underscores '_'" + doAssert name[0] != '_' and name[^1] != '_', errmsg % ["leading/trailing underscores '_'", "--prefix or --suffix"] - doAssert (not name.contains("__")): errmsg & " consecutive underscores '_'" + doAssert (not name.contains("__")): errmsg % ["consecutive underscores '_'", "--replace"] # Cannot blank out symbols which are fields or params # From 865ac56c20c6683b2a352aec6263dc9c817899c5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 10 May 2020 01:49:05 -0500 Subject: [PATCH 134/255] Fix lib version regex --- nimterop.nimble | 6 ++---- nimterop/build.nim | 8 ++++---- tests/getheader.nims | 10 +++++----- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index d60224f..21221a7 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -82,10 +82,8 @@ task test, "Test": # getHeader tests withDir("tests"): exec "nim e getheader.nims" - when not defined(Windows): - # Skip on Windows since very slow - if not existsEnv("APPVEYOR"): - exec "nim e wrappers.nims" + if not existsEnv("APPVEYOR"): + exec "nim e wrappers.nims" docsTask() diff --git a/nimterop/build.nim b/nimterop/build.nim index ece2e33..5978074 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -859,11 +859,11 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT proc getDynlibExt(): string = when defined(Windows): - result = ".dll" + result = "[0-9.\\-]*\\.dll" elif defined(linux) or defined(FreeBSD): - result = ".so[0-9.]*" + result = "\\.so[0-9.]*" elif defined(macosx): - result = ".dylib[0-9.]*" + result = "\\.dylib[0-9.]*" var gDefines {.compileTime.} = initTable[string, string]() @@ -1007,7 +1007,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta preBuild = newIdentNode(name & "PreBuild") # Regex for library search - lre = "(lib)?$1[_-]?(static)?[0-9.\\-]*\\" + lre = "(lib)?$1[_-]?(static)?" # If -d:xxx set with setDefines() stdVal = gDefines.hasKey(stdStr) diff --git a/tests/getheader.nims b/tests/getheader.nims index be22967..5d3b427 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -35,11 +35,6 @@ when defined(posix): testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) - # dl - remove from Windows to save some time - testCall(cmd & " -d:lzmaDL" & lrcmd, "Need version", 1) - testCall(cmd & " -d:lzmaDL -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) - testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0, delete = false) - # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) @@ -48,6 +43,11 @@ testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) testCall(cmd & " -d:zlibGit -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0) testCall(cmd & " -d:zlibGit -d:zlibStatic -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0, delete = false) +# dl +testCall(cmd & " -d:lzmaDL" & lrcmd, "Need version", 1) +testCall(cmd & " -d:lzmaDL -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) +testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0, delete = false) + # dl testCall(cmd & " -d:zlibDL -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0) testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0, delete = false) From 3719607355d7041d8f258280395b4830c9765f54 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 10 May 2020 10:42:01 -0600 Subject: [PATCH 135/255] Don't call bash for unix because it can mess up the configure script --- nimterop/build.nim | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 5978074..48557f3 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -482,8 +482,12 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / i): echo "# Running autogen.sh" - echoDebug execAction( - &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output + when defined(unix): + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output + else: + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output break @@ -499,8 +503,12 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / "configure"): echo "# Running configure " & flags - var - cmd = &"cd {path.sanitizePath} && bash ./configure" + when defined(unix): + var + cmd = &"cd {path.sanitizePath} && ./configure" + else: + var + cmd = &"cd {path.sanitizePath} && bash ./configure" if flags.len != 0: cmd &= &" {flags}" From f693c3e30af162c2a2f377d8b540fe9c34e248fb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 10 May 2020 20:08:07 -0500 Subject: [PATCH 136/255] Fix #209 - cache return val, fix nimcache dir --- nimterop/build.nim | 11 ++++++++--- nimterop/nimconf.nim | 21 ++++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 48557f3..b02132d 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -91,18 +91,22 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, # Else cache for preserving functionality in nimsuggest and nimcheck let hash = (ccmd & cacheKey).hash().abs() - cacheFile = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash & ".txt" + cachePath = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash + cacheFile = cachePath & ".txt" + retFile = cachePath & "_ret.txt" when defined(nimsuggest) or defined(nimcheck): # Load results from cache file if generated in previous run - if fileExists(cacheFile): + if fileExists(cacheFile) and fileExists(retFile): result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() elif die: doAssert false, "Results not cached - run nim c/cpp at least once\n" & ccmd else: - if cache and fileExists(cacheFile) and not compileOption("forceBuild"): + if cache and fileExists(cacheFile) and fileExists(retFile) and not compileOption("forceBuild"): # Return from cache when requested result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() else: # Execute command and store results in cache (result.output, result.ret) = gorgeEx(ccmd) @@ -113,6 +117,7 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, let flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}") cacheFile.writeFile(result.output) + retFile.writeFile($result.ret) else: # Used by toast (result.output, result.ret) = execCmdEx(ccmd) diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index dca68a8..2a54d7e 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -58,12 +58,24 @@ proc getProjectDir*(): string = else: discard +proc stripName(path, projectName: string): string = + # Remove `pname_d|r` tail from path + let + (head, tail) = path.splitPath() + if projectName in tail: + result = head + else: + result = path + proc getNimcacheDir*(projectDir = ""): string = ## Get nimcache directory for current compilation or specified `projectDir` when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): # Get value at compile time from `std/compilesettings` - result = querySetting(SingleValueSetting.nimcacheDir) + result = stripName( + querySetting(SingleValueSetting.nimcacheDir), + querySetting(SingleValueSetting.projectName) + ) else: discard @@ -78,12 +90,7 @@ proc getNimcacheDir*(projectDir = ""): string = dumpJson = getJson(projectDir) if dumpJson != nil and dumpJson.hasKey("nimcache"): - result = dumpJson["nimcache"].getStr() - let - (head, tail) = result.splitPath() - if "dummy" in tail: - # Remove `dummy_d` subdir when default nimcache - result = head + result = stripName(dumpJson["nimcache"].getStr(), "dummy") # Set to OS defaults if not detectable if result.len == 0: From b14eedb541d76aa08066dbc3527ef89ce968e7c2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 May 2020 11:38:09 -0500 Subject: [PATCH 137/255] Support Bool, print debug filename --- nimterop/getters.nim | 5 ++++- nimterop/toast.nim | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f1d1866..f834370 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -103,7 +103,10 @@ const # floating point "float": "cfloat", "double": "cdouble", - "long double": "clongdouble" + "long double": "clongdouble", + + # Misc Nim types + "Bool": "bool" }.toTable() # Nim type names that shouldn't need to be wrapped again diff --git a/nimterop/toast.nim b/nimterop/toast.nim index d74838a..a4fe6fa 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -114,6 +114,9 @@ proc main( doAssert gState.outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}" + if gState.debug: + echo &"# Writing output to {outputFile}\n" + # Process grammar into AST let astTable = From 7daf32d654a2ef0d8751e486f16b306ee5cc22e4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 May 2020 13:08:54 -0500 Subject: [PATCH 138/255] Add reorder pragma for #209, cleanup header pragma --- nimterop/ast2.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 1230e06..d1e691b 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1718,13 +1718,9 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = dynPragma: PNode if not gState.noHeader: - # Path to header const - gState.constSection.add gState.newConstDef( - root, fname = gState.currentHeader, fval = '"' & fullpath & '"') - # {.pragma: impnameHdr, header: "xxx".} hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) - gState.addPragma(root, hdrPragma, "header", gState.getIdent(gState.currentHeader)) + gState.addPragma(root, hdrPragma, "header", newStrNode(nkStrLit, fullpath)) if gState.dynlib.nBl: # {.pragma: impnameDyn, dynlib: libname.} @@ -1737,6 +1733,9 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = if not dynPragma.isNil: gState.pragmaSection.add dynPragma + # Add `{.experimental: "codeReordering".} for #206 + gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) + proc printNimHeader*(gState: State) = # Top level output with context info gecho """# Generated at $1 @@ -1793,9 +1792,9 @@ proc printNim*(gState: State) = # Create output to Nim using Nim compiler renderer var tree = newNode(nkStmtList) + tree.add gState.pragmaSection tree.add gState.enumSection tree.add gState.constSection - tree.add gState.pragmaSection tree.add gState.typeSection tree.add gState.varSection tree.add gState.procSection From ecb5f2ede6bbc793c81ce3bc0f1ea973deb95d2a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 May 2020 14:31:54 -0500 Subject: [PATCH 139/255] v0.5.1 --- nimterop.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 21221a7..6a5cced 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.0" +version = "0.5.1" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -87,4 +87,4 @@ task test, "Test": docsTask() - echo readFile("tests/timeit.txt") \ No newline at end of file + echo readFile("tests/timeit.txt") From 8d4866160ef91084ea13b526694e56ce9c805dd2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 May 2020 16:49:44 -0500 Subject: [PATCH 140/255] Fix #196 - allow remapping of types --- CHANGES.md | 2 ++ README.md | 1 + nimterop/getters.nim | 2 +- nimterop/globals.nim | 2 ++ nimterop/toast.nim | 17 ++++++++++++++--- tests/include/tast2.h | 12 ++++++++++++ tests/tast2.nim | 7 ++++++- 7 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ef66c79..01080a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -56,6 +56,8 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 - `toast` now includes `--replace | -G` to manipulate identifier names beyond `--prefix` and `--suffix`. `-G:X=Y` replaces X with Y and `-G:@_[_]+=_` replaces multiple `_` with a single instance using the `@` prefix to enable regular expressions. +- `toast` also includes `--typeMap | -T` to map C types to another type. E.g. `--typeMap:GLint64=int64` generates a wrapper where all instances of `GLint64` are remapped to the Nim type `int64` and `GLint64` is not defined. (since v0.5.2) + - Nimterop is now able to detect Nim configuration of projects and can better handle cases where defaults such as `nimcacheDir` or `nimblePath` are overridden. This especially enables better interop with workflows that do not depend on Nimble. [#151][i151] [#153][i153] - Nimterop defaults to `cmake`, followed by `autoconf` for building libraries with `getHeader()`. It is now possible to change the order of discovery with the `buildType` value. [#200][i200] diff --git a/README.md b/README.md index 869ac4e..11952e8 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ Options: -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers -O=, --symOverride= strings {} skip generating specified symbols + -T=, --typeMap= strings {} map instances of type X to Y - e.g. ABC=cint ``` ## Why nimterop diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f834370..e600b18 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -28,7 +28,7 @@ yield""".split(Whitespace).toHashSet() # Types related -const +var gTypeMap* = { # char "char": "cchar", diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 05f507c..46203ea 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,6 +76,8 @@ type # `--replace | -G` replacement rules for identifiers suffix*: seq[string] # `--suffix` strings to strip from end of identifiers symOverride*: seq[string] # `cSkipSymbol()`, `cOverride()` and `--symOverride | -O` symbols to skip during wrapping + typeMap*: TableRef[string, string] + # `--typeMap | -T` to map instances of type X to Y - e.g. ABC=cint # cimport.nim specific compile*: seq[string] # `cCompile()` list of files already processed diff --git a/nimterop/toast.nim b/nimterop/toast.nim index a4fe6fa..3a58749 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -1,4 +1,4 @@ -import os, osproc, strformat, strutils, tables, times +import os, osproc, sets, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] @@ -51,6 +51,7 @@ proc main( stub = false, suffix: seq[string] = @[], symOverride: seq[string] = @[], + typeMap: seq[string] = @[], source: seq[string] ) = @@ -93,6 +94,14 @@ proc main( value = if nv.len == 2: nv[1] else: "" gState.replace[name] = value + # typeMap => getters.gTypeMap + for i in typeMap.getSplitComma(): + let + nv = i.split("=", maxsplit = 1) + doAssert nv.len == 2, "`--typeMap` requires X=Y format" + gTypeMap[nv[0]] = nv[1] + gTypeMapValues.incl nv[1] + if pluginSourcePath.nBl: gState.loadPlugin(pluginSourcePath) @@ -217,7 +226,8 @@ when isMainModule: "source" : "C/C++ source/header", "stub": "stub out undefined type references as objects", "suffix": "strip suffix from identifiers", - "symOverride": "skip generating specified symbols" + "symOverride": "skip generating specified symbols", + "typeMap": "map instances of type X to Y - e.g. ABC=cint" }, short = { "check": 'k', "convention": 'C', @@ -238,5 +248,6 @@ when isMainModule: "replace": 'G', "stub": 's', "suffix": 'F', - "symOverride": 'O' + "symOverride": 'O', + "typeMap": 'T' }) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 878a3a2..bbb3315 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -243,6 +243,12 @@ static inline int sitest1(int f1) { return f1 * 2; } +// Issue #196 +typedef int MyInt; +struct TestMyInt { + MyInt f1; +}; + // DUPLICATES @@ -439,6 +445,12 @@ static inline int sitest1(int f1) { return f1 * 2; } +// Issue #196 +typedef int MyInt; +struct TestMyInt { + MyInt f1; +}; + #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index bd040e2..4fd16cc 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -35,7 +35,7 @@ cOverride: type A1* = A0 -cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE" & flags) +cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags) proc getPragmas(n: NimNode): HashSet[string] = # Find all pragmas in AST, return as "name" or "name:value" in set @@ -478,3 +478,8 @@ checkPragmas(nested, pHeaderImpBy) when not defined(NOHEADER): assert sitest1(5) == 10 assert sitest1(10) == 20 + +when declared(MyInt): + assert false, "MyInt is defined!" +testFields(TestMyInt, "f1!cint") +checkPragmas(TestMyInt, pHeaderBy, isType = false) \ No newline at end of file From 8e052269c996ac1e50bce6643604ab8128e921c3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 May 2020 23:58:18 -0500 Subject: [PATCH 141/255] v0.5.2 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 6a5cced..2a8c269 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.1" +version = "0.5.2" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 37fa2ad7fb69553ec37d5e4b46988feaca7f80f1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 12 May 2020 22:01:42 -0500 Subject: [PATCH 142/255] Add CLI file feature --- CHANGES.md | 3 +++ README.md | 4 ++-- nimterop.nimble | 4 ++-- nimterop/ast2.nim | 14 ++------------ nimterop/toast.nim | 26 ++++++++++++++++++++++++-- tests/toast.cfg | 2 ++ 6 files changed, 35 insertions(+), 18 deletions(-) create mode 100644 tests/toast.cfg diff --git a/CHANGES.md b/CHANGES.md index 01080a6..ea3f427 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -58,6 +58,8 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 - `toast` also includes `--typeMap | -T` to map C types to another type. E.g. `--typeMap:GLint64=int64` generates a wrapper where all instances of `GLint64` are remapped to the Nim type `int64` and `GLint64` is not defined. (since v0.5.2) +- CLI flags can now be specified one or more per line in a file and path provided to `toast`. They will be expanded in place. [#196][i196] (since v0.5.3) + - Nimterop is now able to detect Nim configuration of projects and can better handle cases where defaults such as `nimcacheDir` or `nimblePath` are overridden. This especially enables better interop with workflows that do not depend on Nimble. [#151][i151] [#153][i153] - Nimterop defaults to `cmake`, followed by `autoconf` for building libraries with `getHeader()`. It is now possible to change the order of discovery with the `buildType` value. [#200][i200] @@ -78,5 +80,6 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 [i174]: https://github.com/nimterop/nimterop/issues/174 [i176]: https://github.com/nimterop/nimterop/issues/176 [i181]: https://github.com/nimterop/nimterop/issues/181 +[i196]: https://github.com/nimterop/nimterop/issues/196 [i197]: https://github.com/nimterop/nimterop/issues/197 [i200]: https://github.com/nimterop/nimterop/issues/200 \ No newline at end of file diff --git a/README.md b/README.md index 11952e8..48822ac 100644 --- a/README.md +++ b/README.md @@ -195,14 +195,14 @@ Nimterop also provides a [docs](https://nimterop.github.io/nimterop/docs.html) A ### Command line API -The `toast` binary can also be used directly on the CLI, similar to `c2nim`. The `cPlugin()` interface +The `toast` binary can also be used directly on the CLI, similar to `c2nim`. These flags can be specified on the command line or via a file, one or more flags per line, and the path provided to `toast` instead, or a combination. The file contents will be expanded in place. Note: unlike the wrapper API, the `-p | --preprocess` flag is not enabled by default but is *highly* recommended. ``` > toast -h Usage: - main [optional-params] C/C++ source/header + main [optional-params] C/C++ source/header(s) and command line file(s) Options: -h, --help print this cligen-erated help --help-syntax advanced: prepend,plurals,.. diff --git a/nimterop.nimble b/nimterop.nimble index 2a8c269..4ec024c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -59,8 +59,8 @@ task test, "Test": execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\"" execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" - execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" - execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h" + execCmd "./nimterop/toast tests/toast.cfg tests/include/toast.h" + execCmd "./nimterop/toast tests/toast.cfg -f:ast2 tests/include/toast.h" execTest "tests/tpcre.nim" execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index d1e691b..4a9b048 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -139,7 +139,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gecho &"# const '{origname}' is duplicate, skipped" else: - gecho &"# const '{origname}' has invalid value '{val}'" + gecho &"# const '{origname}' has unsupported value '{val}'" gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = @@ -1736,19 +1736,9 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # Add `{.experimental: "codeReordering".} for #206 gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) -proc printNimHeader*(gState: State) = - # Top level output with context info - gecho """# Generated at $1 -# Command line: -# $2 $3 - -{.hint[ConvFromXtoItselfNotNeeded]: off.} - -import nimterop/types -""" % [$now(), getAppFilename(), commandLineParams().join(" ")] - proc initNim*(gState: State) = # Initialize for parseNim() one time + gecho "import nimterop/types\n" # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 3a58749..b1f910c 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -141,7 +141,6 @@ proc main( elif source.nBl: # Print source after preprocess or Nim output if gState.pnim: - gState.printNimHeader() gState.initNim() for src in source: gState.process(src.expandSymlinkAbs(), astTable) @@ -199,6 +198,29 @@ proc main( if check and output.len == 0: stdout.write outputFile.readFile() +proc mergeParams(cmdNames: seq[string], cmdLine = commandLineParams()): seq[string] = + # Load command-line params from `source` if it is a .cfg file + if cmdNames.len != 0: + # https://github.com/c-blake/cligen/issues/149 + for param in cmdLine: + if param.fileExists() and param.splitFile().ext == ".cfg": + echo &"# Loading flags from '{param}'" + for line in param.readFile().splitLines(): + let + line = line.strip() + if line.len > 1 and line[0] != '#': + result.add line.parseCmdLine() + else: + result.add param + + if result.len != 0 and "-h" notin result and "--help" notin result: + echo &"""# Generated @ {$now()} +# Command line: +# {getAppFilename()} {result.join(" ")} +""" + else: + result = cmdLine + when isMainModule: # Setup cligen command line help and short flags import cligen @@ -223,7 +245,7 @@ when isMainModule: "preprocess": "run preprocessor on header", "recurse": "process #include files", "replace": "replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex", - "source" : "C/C++ source/header", + "source" : "C/C++ source/header(s) and command line file(s)", "stub": "stub out undefined type references as objects", "suffix": "strip suffix from identifiers", "symOverride": "skip generating specified symbols", diff --git a/tests/toast.cfg b/tests/toast.cfg new file mode 100644 index 0000000..8c37f3c --- /dev/null +++ b/tests/toast.cfg @@ -0,0 +1,2 @@ +--preprocess +-nk -E=_ \ No newline at end of file From fc587fae037c0443cc1a2ca64685ba1d3b9d62d3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 13 May 2020 14:36:38 -0500 Subject: [PATCH 143/255] v0.5.3 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 4ec024c..187b0ce 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.2" +version = "0.5.3" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 32c98435486cff69f7ecbcfb2be3c8f1739fe177 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 16:22:53 -0600 Subject: [PATCH 144/255] Fix undefined identifiers resulting in incorrect types --- nimterop/ast2.nim | 14 ++++++++++++-- tests/include/tast2.h | 15 +++++++++++++++ tests/tast2.nim | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4a9b048..574870f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -573,8 +573,12 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) pident = gState.getIdent(pname, pinfo, exported) result.add pident - result.add gState.getTypeArray(node[i], tident, name) - result.add newNode(nkEmpty) + let tyArray = gState.getTypeArray(node[i], tident, name) + if tyArray.kind != nkNone: + result.add tyArray + result.add newNode(nkEmpty) + else: + result = nil else: result = nil @@ -955,6 +959,8 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] + else: + result = newNode(nkNone) elif cnode.len == 1: # type name[] = UncheckedArray[type] result = gState.newArrayTree(cnode, result) @@ -985,6 +991,10 @@ proc addTypeArray(gState: State, node: TSNode) = name = typeDef.getIdentName() typ = gState.getTypeArray(node[i], tident, name) + if typ.kind == nkNone: + gecho (&"{gState.getNodeVal(node)} skipped").getCommented() + continue + typeDef.add typ # type X* = [ptr] array[x, [ptr] Y] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index bbb3315..f28c003 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -49,6 +49,21 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) +// const not supported yet +const int SOME_CONST = 8; + +struct some_struct_s +{ + int x; +}; + +struct parent_struct_s +{ + struct some_struct_s s[SOME_CONST]; +}; + +typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; + struct A0; struct A1 {}; typedef struct A2; diff --git a/tests/tast2.nim b/tests/tast2.nim index 4fd16cc..058ad0b 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -155,6 +155,9 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) +assert not compiles(parent_struct_s().s) +assert not defined(SOME_ARRAY) + assert A0 is object testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) From 1df2943cf80147a7d32f1edff5572d92162c663e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 16:32:44 -0600 Subject: [PATCH 145/255] Change to static const --- tests/include/tast2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index f28c003..d39e8ea 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -50,7 +50,7 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) // const not supported yet -const int SOME_CONST = 8; +static const int SOME_CONST = 8; struct some_struct_s { From ed26911436a2107d287a8f1d4f4df7727d6de53d Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 17:18:47 -0600 Subject: [PATCH 146/255] Disable windows check --- tests/include/tast2.h | 5 ++++- tests/tast2.nim | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index d39e8ea..57c060e 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -49,8 +49,10 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) +// disable for windows for now +#ifndef _WIN32 // const not supported yet -static const int SOME_CONST = 8; +const int SOME_CONST = 8; struct some_struct_s { @@ -63,6 +65,7 @@ struct parent_struct_s }; typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; +#endif struct A0; struct A1 {}; diff --git a/tests/tast2.nim b/tests/tast2.nim index 058ad0b..f132609 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -155,8 +155,9 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) -assert not compiles(parent_struct_s().s) -assert not defined(SOME_ARRAY) +when not defined(windows): + assert not compiles(parent_struct_s().s) + assert not defined(SOME_ARRAY) assert A0 is object testFields(A0, "f1!cint") From 9895303578c5313a775b9226d051dfe63905495a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 17:29:26 -0600 Subject: [PATCH 147/255] Make undefined constants keep types --- nimterop/ast2.nim | 29 ++++++++++++----------------- nimterop/toast.nim | 2 ++ tests/include/tast2.h | 12 +++++------- tests/tast2.nim | 6 +++--- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 574870f..3b708b5 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -574,11 +574,8 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt pident = gState.getIdent(pname, pinfo, exported) result.add pident let tyArray = gState.getTypeArray(node[i], tident, name) - if tyArray.kind != nkNone: - result.add tyArray - result.add newNode(nkEmpty) - else: - result = nil + result.add tyArray + result.add newNode(nkEmpty) else: result = nil @@ -953,14 +950,16 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo for i in 0 ..< acount: if cnode.len == 2: # type name[X] => array[X, type] - let - # Size of array could be a Nim expression - size = gState.parseCExpression(gState.getNodeVal(cnode[1])) - if size.kind != nkNone: - result = gState.newArrayTree(cnode, result, size) - cnode = cnode[0] - else: - result = newNode(nkNone) + var size: PNode + let cnodeVal = gState.getNodeVal(cnode[1]) + # Size of array could be a Nim expression + size = gState.parseCExpression(cnodeVal) + if size.kind == nkNone: + # If the size could not be parsed, leave it alone + size = gState.getIdent(cnodeVal) + + result = gState.newArrayTree(cnode, result, size) + cnode = cnode[0] elif cnode.len == 1: # type name[] = UncheckedArray[type] result = gState.newArrayTree(cnode, result) @@ -991,10 +990,6 @@ proc addTypeArray(gState: State, node: TSNode) = name = typeDef.getIdentName() typ = gState.getTypeArray(node[i], tident, name) - if typ.kind == nkNone: - gecho (&"{gState.getNodeVal(node)} skipped").getCommented() - continue - typeDef.add typ # type X* = [ptr] array[x, [ptr] Y] diff --git a/nimterop/toast.nim b/nimterop/toast.nim index b1f910c..f54aa67 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,6 +4,8 @@ import "."/treesitter/[api, c, cpp] import "."/[ast, ast2, build, globals, getters, grammar, tshelp] +{.passC: "-DNIMTEROP".} + proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 57c060e..b8433f9 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -42,17 +42,16 @@ extern "C" { #define REG_STR "regular string" #define NOTSUPPORTEDSTR "not a " REG_STR -#define NULLCHAR '\0'/* comments should not break things*/ -#define OCTCHAR '\012' // nor should this comment +#define NULLCHAR '\0' +#define OCTCHAR '\012' #define HEXCHAR '\xFE' #define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) -// disable for windows for now -#ifndef _WIN32 -// const not supported yet -const int SOME_CONST = 8; +#ifdef NIMTEROP +#define SOME_CONST 8 +#endif struct some_struct_s { @@ -65,7 +64,6 @@ struct parent_struct_s }; typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; -#endif struct A0; struct A1 {}; diff --git a/tests/tast2.nim b/tests/tast2.nim index f132609..8deb99e 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -35,6 +35,7 @@ cOverride: type A1* = A0 +cDefine("SOME_CONST=100") cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags) proc getPragmas(n: NimNode): HashSet[string] = @@ -155,9 +156,8 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) -when not defined(windows): - assert not compiles(parent_struct_s().s) - assert not defined(SOME_ARRAY) +assert typeof(parent_struct_s().s) is array[100, some_struct_s] +assert typeof(SOME_ARRAY) is array[100, some_struct_s] assert A0 is object testFields(A0, "f1!cint") From 1a7860d8f4ac5d2d2d453e75e05200cb34849954 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 18:31:07 -0600 Subject: [PATCH 148/255] Get rid of extra var --- nimterop/ast2.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 3b708b5..0d0883c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -573,8 +573,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) pident = gState.getIdent(pname, pinfo, exported) result.add pident - let tyArray = gState.getTypeArray(node[i], tident, name) - result.add tyArray + result.add gState.getTypeArray(node[i], tident, name) result.add newNode(nkEmpty) else: result = nil From 2098f6a471784b5c87c678ec9d79faf2cc467199 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 22:19:37 -0600 Subject: [PATCH 149/255] Preserve type array ast --- nimterop/ast2.nim | 5 +---- nimterop/exprparser.nim | 14 ++++++++------ nimterop/globals.nim | 3 +++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 0d0883c..389d371 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -952,10 +952,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo var size: PNode let cnodeVal = gState.getNodeVal(cnode[1]) # Size of array could be a Nim expression - size = gState.parseCExpression(cnodeVal) - if size.kind == nkNone: - # If the size could not be parsed, leave it alone - size = gState.getIdent(cnodeVal) + size = gState.parseCExpression(cnodeVal, skipIdentValidation = true) result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b0c34c1..e5e2150 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,14 +45,14 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - if identName notin gState.skippedSyms: + if gState.currentExprSkipIdentValidation or identName notin gState.skippedSyms: var ident = identName if ident != "_": # Process the identifier through cPlugin ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: result = gState.getIdent(ident) - elif ident.nBl and ident in gState.constIdentifiers: + elif gState.currentExprSkipIdentValidation or ident.nBl and ident in gState.constIdentifiers: if gState.currentTyCastName.nBl: ident = ident & "." & gState.currentTyCastName result = gState.getIdent(ident) @@ -591,7 +591,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = decho "NODE RESULT: ", result -proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = +proc parseCExpression*(gState: State, codeRoot: TSNode): PNode = ## Parse a c expression from a root ts node # This var is used for keeping track of the type of the first @@ -607,14 +607,16 @@ proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = decho "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) -proc parseCExpression*(gState: State, code: string, name = ""): PNode = +proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidation = false): PNode = ## Convert the C string to a nim PNode tree gState.currentExpr = code gState.currentTyCastName = name + gState.currentExprSkipIdentValidation = skipIdentValidation withCodeAst(gState.currentExpr, gState.mode): - result = gState.parseCExpression(root, name) + result = gState.parseCExpression(root) # Clear the state gState.currentExpr = "" - gState.currentTyCastName = "" \ No newline at end of file + gState.currentTyCastName = "" + gState.currentExprSkipIdentValidation = false \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 46203ea..039dd3c 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -114,6 +114,9 @@ type # Used for the exprparser.nim module currentExpr*, currentTyCastName*: string + # Controls whether or not the current expression + # should validate idents against currently defined idents + currentExprSkipIdentValidation*: bool # Legacy AST fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From faab2924c8a6a48df65a4ee74363eb822c2c768f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 22:36:40 -0600 Subject: [PATCH 150/255] Cleanup code --- nimterop/ast2.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 389d371..dff355e 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -949,10 +949,9 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo for i in 0 ..< acount: if cnode.len == 2: # type name[X] => array[X, type] - var size: PNode - let cnodeVal = gState.getNodeVal(cnode[1]) - # Size of array could be a Nim expression - size = gState.parseCExpression(cnodeVal, skipIdentValidation = true) + let + # Size of array could be a Nim expression + size = gState.parseCExpression(gState.getNodeVal(cnode[1]), skipIdentValidation = true) result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] From ffd6b9b5eb6be220b05178438aa622cda047d9d5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 14 May 2020 17:23:43 -0600 Subject: [PATCH 151/255] Rename currentExprSkipIdentValidation -> skipIdentValidation --- nimterop/exprparser.nim | 8 ++++---- nimterop/globals.nim | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e5e2150..b089fd2 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,14 +45,14 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - if gState.currentExprSkipIdentValidation or identName notin gState.skippedSyms: + if gState.skipIdentValidation or identName notin gState.skippedSyms: var ident = identName if ident != "_": # Process the identifier through cPlugin ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: result = gState.getIdent(ident) - elif gState.currentExprSkipIdentValidation or ident.nBl and ident in gState.constIdentifiers: + elif gState.skipIdentValidation or ident.nBl and ident in gState.constIdentifiers: if gState.currentTyCastName.nBl: ident = ident & "." & gState.currentTyCastName result = gState.getIdent(ident) @@ -611,7 +611,7 @@ proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidati ## Convert the C string to a nim PNode tree gState.currentExpr = code gState.currentTyCastName = name - gState.currentExprSkipIdentValidation = skipIdentValidation + gState.skipIdentValidation = skipIdentValidation withCodeAst(gState.currentExpr, gState.mode): result = gState.parseCExpression(root) @@ -619,4 +619,4 @@ proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidati # Clear the state gState.currentExpr = "" gState.currentTyCastName = "" - gState.currentExprSkipIdentValidation = false \ No newline at end of file + gState.skipIdentValidation = false \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 039dd3c..05284a9 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -116,7 +116,7 @@ type currentExpr*, currentTyCastName*: string # Controls whether or not the current expression # should validate idents against currently defined idents - currentExprSkipIdentValidation*: bool + skipIdentValidation*: bool # Legacy AST fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From 5895db1d18e1d3c9f1e502f0a68f671d4e53e02b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 16:58:07 -0600 Subject: [PATCH 152/255] Remove regex from comment processing --- nimterop/tshelp.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 762cf7c..9565a1f 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,7 +1,5 @@ import sets, strformat, strutils -import regex - import "."/[getters, globals] import "."/treesitter/[api, c, cpp] @@ -273,8 +271,14 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).strip() - result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") - result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() + result = result.multiReplace( + { + "/**": "", "**/": "", "/*": "", + "*/": "", "/*": "", "//": "", + "\n": "\n ", "`": "" + } + # need to replace this last otherwise it supercedes other replacements + ).replace(" *", "").strip() proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` From 985741cc8de35ea36933ddb845f93316288c30b8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 17:52:21 -0600 Subject: [PATCH 153/255] Remove more regex from exprparser --- nimterop/exprparser.nim | 61 ++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b089fd2..b2b8c2c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros, sets +import strformat, strutils, macros, sets, sequtils import regex @@ -32,6 +32,10 @@ import "."/[globals, getters, comphelp, tshelp] type ExprParseError* = object of CatchableError +const + CharRegStr = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + CharRegex = re(CharRegStr) + template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) @@ -138,13 +142,13 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = flags: TNodeFlags # I realize these regex are wasteful on performance, but # couldn't come up with a better idea. - if number.contains(re"0[xX]"): + if number.startsWith("0X") or number.startsWith("0x"): val = parseHexInt(number) flags = {nfBase16} - elif number.contains(re"0[bB]"): + elif number.startsWith("0B") or number.startsWith("0b"): val = parseBinInt(number) flags = {nfBase2} - elif number.contains(re"0[oO]"): + elif number.startsWith("0O") or number.startsWith("0o"): val = parseOctInt(number) flags = {nfBase8} else: @@ -186,25 +190,36 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val + var + prefix: string + number = nodeVal + suffix: string - var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" - let found = nodeVal.find(reg, match) - if found: - let - prefix = if match.group(0).len > 0: nodeVal[match.group(0)[0]] else: "" - number = nodeVal[match.group(1)[0]] - suffix = nodeVal[match.group(2)[0]] + const + singleEndings = ["u", "l", "U", "L"] + doubleEndings = ["ul", "UL", "ll", "LL"] + tripleEndings = ["ull", "ULL"] - result = getNumNode(number, suffix) + if number.startsWith("-"): + number = number[1 ..< number.len] + prefix = "-" + if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^3 .. ^1] + number = number[0 ..< ^3] + elif doubleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^2 .. ^1] + number = number[0 ..< ^2] + elif singleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = $number[number.len - 1] + number = number[0 ..< ^1] - if result.kind != nkNone and prefix == "-": - result = nkPrefix.newTree( - gState.getIdent("-"), - result - ) - else: - raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") + result = getNumNode(number, suffix) + + if result.kind != nkNone and prefix == "-": + result = nkPrefix.newTree( + gState.getIdent("-"), + result + ) proc processCharacterLiteral(gState: State, node: TSNode): PNode = # Input => 'G' @@ -234,13 +249,9 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] - const - str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" - reg = re(str) - # Convert the c string escape sequences/etc to Nim chars var nimStr = newStringOfCap(nodeVal.len) - for m in strVal.findAll(reg): + for m in strVal.findAll(CharRegex): nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) result = newStrNode(nkStrLit, nimStr) From 11f84d4d8075d96aed623d09972a79f2702486a8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 18 May 2020 11:22:54 -0500 Subject: [PATCH 154/255] var support, interleaved #defines, fix ast1 default, misc performance --- nimterop/ast2.nim | 501 ++++++++++++++++++++++------------------ nimterop/exprparser.nim | 3 +- nimterop/getters.nim | 35 +-- nimterop/globals.nim | 2 +- nimterop/toast.nim | 8 +- nimterop/tshelp.nim | 2 +- tests/rsa.nim | 3 +- 7 files changed, 314 insertions(+), 240 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dff355e..3d52e02 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,7 +1,5 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import options as opts - import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api @@ -136,6 +134,8 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = result.add valident[0] else: result.add valident + # In case symbol was skipped earlier + gState.skippedSyms.excl origname else: gecho &"# const '{origname}' is duplicate, skipped" else: @@ -251,7 +251,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: (tname, torigname, info) = if not atom.isNil: - gState.getNameInfo(node.getAtom(), kind) + gState.getNameInfo(atom, kind) else: ("", "", gState.getLineInfo(node)) @@ -342,46 +342,42 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: result.add prident result.add newNode(nkEmpty) elif kind == nskVar: - # var name* {.importc: "abc".} + # Used by `addVar()` for regular vars and proc vars # - # nkIdentDefs( - # nkPragmaExpr( - # nkPostfix( - # nkIdent("*"), - # nkIdent(name) - # ), - # nkPragma( - # nkExprColonExpr( - # nkIdent("importc"), - # nkStrLit("abc") - # ) + # name* {.importc: "abc".} + # + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("abc") # ) # ) # ) - let - prident = block: - var - prident: PNode - # Add {.importc.} pragma - if name != origname: - # Name changed - prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) - else: - prident = gState.newPragmaExpr(node, ident, "importc") + result = block: + var + prident: PNode + # Add {.importc.} pragma + if name != origname: + # Name changed + prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) + else: + prident = gState.newPragmaExpr(node, ident, "importc") - if gState.dynlib.nBl: - # Add {.dynlib.} - gState.addPragma(node, prident[1], gState.impShort & "Dyn") - elif not gState.noHeader: - # Add {.header.} - gState.addPragma(node, prident[1], gState.impShort & "Hdr") + if gState.dynlib.nBl: + # Add {.dynlib.} + gState.addPragma(node, prident[1], gState.impShort & "Dyn") + elif not gState.noHeader: + # Add {.header.} + gState.addPragma(node, prident[1], gState.impShort & "Hdr") - if pragmas.nBl: - gState.addPragma(node, prident[1], pragmas) - prident - - result = newNode(nkIdentDefs) - result.add prident + if pragmas.nBl: + gState.addPragma(node, prident[1], pragmas) + prident elif kind == nskProc: # name* # @@ -426,12 +422,39 @@ proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNode proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode -iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = - # Create nkIdentDefs tree for specified proc parameter or object field +proc getTypeAndStart(gState: State, name: string, node: TSNode, ftname = ""): + tuple[tname: string, tinfo: TLineInfo, tident: PNode, start: int] = + # Shortcut to get start node and type info from first node # - # For proc, param should not be `exported` + # `name` is the parent type or proc # - # If `ftname` is set, use it as the type name + # If `ftname` is set, use it as the type name instead + result.start = getStartAtom(node) + + let + # node[start] - param type + (tname0, _, tinfo) = gState.getNameInfo(node[result.start].getAtom(), nskType, parent = name) + + # Override type name + result.tname = + if ftname.nBl: + ftname + else: + tname0 + + result.tinfo = tinfo + result.tident = gState.getIdent(result.tname, tinfo, exported = false) + +proc newIdentDef(gState: State, name: string, node: TSNode, tname: string, tinfo: TLineInfo, tident: PNode, + start, offset, poffset: SomeInteger, exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter, object field or var/proc var + # + # `name` is the parent type or proc - if blank, this is a var so add pragmas + # `tname`, `tinfo` and `tident` are the type info for the node + # `start` is the `node` child where type is located + # `offset` is the nth `node` child where specified param/field/var is located - from `addVar()` + # `poffset` is the param number and used for unnamed params in procs - from `newIdentDefs()` iterator + # `isvar` when true adds pragmas # # pname: [ptr ..] typ # @@ -441,7 +464,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # nkEmpty() # ) # - # For object, field should be exported + # For objects and vars/proc vars, field should be exported # # pname*: [ptr ..] typ # @@ -453,6 +476,125 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # typ, # nkEmpty() # ) + result = newNode(nkIdentDefs) + + let + fdecl = node[offset].firstChildInTree("function_declarator") + afdecl = node[offset].firstChildInTree("abstract_function_declarator") + adecl = node[offset].firstChildInTree("array_declarator") + abst = node[offset].getName() == "abstract_pointer_declarator" + if fdecl.isNil and afdecl.isNil and adecl.isNil: + if abst: + # Only for proc with no named param with pointer type + # Create a param name based on poffset + # + # int func(char *, int **); + let + pname = "a" & $(poffset+1) + pident = gState.getIdent(pname, tinfo, exported) + acount = node[offset].getXCount("abstract_pointer_declarator") + result.add pident + result.add gState.newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + # Named param, simple type + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + + # Bitfield support - typedef struct { int field: 1; }; + prident = + if node.len > offset and node[offset + 1].getName() == "bitfield_clause": + gState.newPragmaExpr(node, pident, "bitsize", + newIntNode(nkIntLit, parseInt(gState.getNodeVal(node[offset + 1].getAtom())))) + else: + pident + + result.add prident + else: + # Vars - `addVar()` + let + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + + result.add pident + + let + count = node[offset].getPtrCount() + if count > 0: + result.add gState.newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + # Named param, function pointer + var + name = name + pident: PNode + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + else: + # Vars - `addVar()` + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + # The var is the parent + name = pident.getIdentName() + + result.add pident + result.add gState.getTypeProc(name, node[offset], node[start]) + result.add newNode(nkEmpty) + elif not afdecl.isNil: + # Only for proc with no named param with function pointer type + # Create a param name based on poffset + # + # int func(int (*)(int *)); + let + pname = "a" & $(poffset+1) + pident = gState.getIdent(pname, tinfo, exported) + procTy = gState.getTypeProc(name, node[offset], node[start]) + result.add pident + result.add procTy + result.add newNode(nkEmpty) + elif not adecl.isNil: + # Named param, array type + var + name = name + pident: PNode + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + else: + # Vars - `addVar()` + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + # The var is the parent + name = pident.getIdentName() + + result.add pident + result.add gState.getTypeArray(node[offset], tident, name) + result.add newNode(nkEmpty) + else: + result = nil + +iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter or object field + # + # `name` is the parent type or proc + # `offset` is the param number and used for unnamed params in procs + # + # For proc, param should not be `exported` + # + # If `ftname` is set, use it as the type name # # Iterator since structs can have multiple comma separated fields for the # same type so can yield multiple results. @@ -460,23 +602,12 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # struct ABC { int w, h; }; # # This is not applicable for procs. - var - start = getStartAtom(node) - let - # node[start] - param type - (tname0, _, tinfo) = gState.getNameInfo(node[start].getAtom(), nskType, parent = name) - - # Override type name - tname = - if ftname.nBl: - ftname - else: - tname0 - - tident = gState.getIdent(tname, tinfo, exported = false) + (tname, tinfo, tident, start0) = gState.getTypeAndStart(name, node, ftname) # Skip qualifiers after type + var + start = start0 while start < node.len - 1 and node[start+1].getName() == "type_qualifier": start += 1 @@ -503,82 +634,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt for i in start+1 ..< node.len: if node[i].getName() == "bitfield_clause": continue - - var - result = newNode(nkIdentDefs) - - let - fdecl = node[i].firstChildInTree("function_declarator") - afdecl = node[i].firstChildInTree("abstract_function_declarator") - adecl = node[i].firstChildInTree("array_declarator") - abst = node[i].getName() == "abstract_pointer_declarator" - if fdecl.isNil and afdecl.isNil and adecl.isNil: - if abst: - # Only for proc with no named param with pointer type - # Create a param name based on offset - # - # int func(char *, int **); - let - pname = "a" & $(offset+1) - pident = gState.getIdent(pname, tinfo, exported) - acount = node[i].getXCount("abstract_pointer_declarator") - result.add pident - result.add gState.newPtrTree(acount, tident) - result.add newNode(nkEmpty) - else: - # Named param, simple type - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - - # Bitfield support - typedef struct { int field: 1; }; - prident = - if node.len > i and node[i + 1].getName() == "bitfield_clause": - gState.newPragmaExpr(node, pident, "bitsize", - newIntNode(nkIntLit, parseInt(gState.getNodeVal(node[i + 1].getAtom())))) - else: - pident - - count = node[i].getPtrCount() - - result.add prident - if count > 0: - result.add gState.newPtrTree(count, tident) - else: - result.add tident - result.add newNode(nkEmpty) - elif not fdecl.isNil: - # Named param, function pointer - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - result.add pident - result.add gState.getTypeProc(name, node[i], node[start]) - result.add newNode(nkEmpty) - elif not afdecl.isNil: - # Only for proc with no named param with function pointer type - # Create a param name based on offset - # - # int func(int (*)(int *)); - let - pname = "a" & $(offset+1) - pident = gState.getIdent(pname, tinfo, exported) - procTy = gState.getTypeProc(name, node[i], node[start]) - result.add pident - result.add procTy - result.add newNode(nkEmpty) - elif not adecl.isNil: - # Named param, array type - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - result.add pident - result.add gState.getTypeArray(node[i], tident, name) - result.add newNode(nkEmpty) - else: - result = nil - - yield result + yield gState.newIdentDef(name, node, tname, tinfo, tident, start, i, offset, exported) proc newFormalParams(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkFormalParams tree for specified params and return type @@ -967,11 +1023,8 @@ proc addTypeArray(gState: State, node: TSNode) = # Add a type of array type decho("addTypeArray()") let - start = getStartAtom(node) + (_, _, tident, start) = gState.getTypeAndStart("addTypeArray", node) - # node[start] = identifier = type name - (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") - tident = gState.getIdent(tname, info, exported = false) commentNodes = gState.getCommentNodes(node) # Could have multiple types, comma separated @@ -1015,6 +1068,7 @@ proc addTypeArray(gState: State, node: TSNode) = proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode = # Create proc type tree # + # `name` is the parent type or proc # `rnode` is the return type let # rnode = identifier = return type name @@ -1368,7 +1422,7 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]] + fieldDeclarations: seq[tuple[fname, forigname, fval, cexpr: string, comment: seq[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1378,7 +1432,8 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() commentNodes = gState.getCommentNodes(en) - fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) + forigname = gState.getNodeVal(atom) + fname = gState.getIdentifier(forigname, nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): var @@ -1391,9 +1446,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]), commentNodes)) + fieldDeclarations.add((fname, forigname, "", gState.getNodeVal(en[1]), commentNodes)) else: - fieldDeclarations.add((fname, fval, none(TSNode), commentNodes)) + fieldDeclarations.add((fname, forigname, fval, "", commentNodes)) fnames.incl fname prev = fname @@ -1403,78 +1458,25 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode, commentNodes) in fieldDeclarations: - var fval = fval - if cexprNode.isSome: - fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name - # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + for (fname, forigname, fval, cexpr, commentNodes) in fieldDeclarations: + let + fval = + if fval.Bl: + "(" & $gState.parseCExpression(cexpr, name) & ")." & name + else: fval + + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + constNode.comment = gState.getCommentsStr(commentNodes) gState.constSection.add constNode + # In case symbol was skipped earlier + gState.skippedSyms.excl forigname # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = - # Add a proc variable - decho("addProcVar()") - let - # node = identifier = name - identDefs = gState.newXIdent(node, kind = nskVar, istype = true) - - if not identDefs.isNil: - let - name = identDefs.getIdentName() - # origname = gState.getNodeVal(node.getAtom()) - - procTy = gState.getTypeProc(name, node, rnode) - - identDefs.add procTy - identDefs.add newNode(nkEmpty) - - # var X* {.importc: "_X": proc(a1: Y, a2: Z): P {.cdecl.} - # - # nkIdentDefs( - # nkPragmaExpr( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkPragma( - # nkExprColonExpr( - # nkIdent("importc"), - # nkStrLit("_X") - # ) - # ) - # ), - # nkProcTy( - # nkFormalParams( - # nkIdent("P"), - # nkIdentDefs( - # nkIdent("a1"), - # nkIdent("Y"), - # nkEmpty() - # ), - # nkIdentDefs( - # nkIdent("a2"), - # nkIdent("Z"), - # nkEmpty() - # ) - # ), - # nkPragma( - # nkIdent("cdecl") - # ) - # ), - # nkEmpty() - # ) - - identDefs.comment = gState.getCommentsStr(commentNodes) - # nkVarSection.add - gState.varSection.add identDefs - - gState.printDebug(identDefs) - proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # @@ -1585,6 +1587,65 @@ proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = gState.printDebug(procDef) +proc addVar(gState: State, node: TSNode, offset: SomeInteger, commentNodes: seq[TSNode]) = + # Add a regular variable + # + # `node` is the `nth` child of (declaration) + # `tnode` is the type node, the first child of (declaration) + decho("addVar()") + let + (tname, tinfo, tident, start) = gState.getTypeAndStart("", node) + + identDefs = gState.newIdentDef("", node, tname, tinfo, tident, start, offset, 0, exported = true) + + if not identDefs.isNil: + # proc var + # + # P (*_X)(Y a1, Z a2); + # + # var X* {.importc: "_X": proc(a1: Y, a2: Z): P {.cdecl.} + # + # nkIdentDefs( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("_X") + # ) + # ) + # ), + # nkProcTy( + # nkFormalParams( + # nkIdent("P"), + # nkIdentDefs( + # nkIdent("a1"), + # nkIdent("Y"), + # nkEmpty() + # ), + # nkIdentDefs( + # nkIdent("a2"), + # nkIdent("Z"), + # nkEmpty() + # ) + # ), + # nkPragma( + # nkIdent("cdecl") + # ) + # ), + # nkEmpty() + # ) + + identDefs.comment = gState.getCommentsStr(commentNodes) + + # nkVarSection.add + gState.varSection.add identDefs + + gState.printDebug(identDefs) + proc addDecl(gState: State, node: TSNode) = # Add a declaration decho("addDecl()") @@ -1598,31 +1659,29 @@ proc addDecl(gState: State, node: TSNode) = commentNodes: seq[TSNode] for i in start+1 ..< node.len: + if node[i].getName() == "comment": + continue + + if firstDecl: + # If it's the first declaration, use the whole node + # to get the comment above/below + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) + if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - if firstDecl: - # If it's the first declaration, use the whole node - # to get the comment above/below - commentNodes = gState.getCommentNodes(node) - firstDecl = false - else: - commentNodes = gState.getCommentNodes(node[i]) - gState.addProcVar(node[i], node[start], commentNodes) + #gState.addProcVar(node[i], node[start], commentNodes) + gState.addVar(node, i, commentNodes) else: # proc - if firstDecl: - # If it's the first declaration, use the whole node - # to get the comment above/below - commentNodes = gState.getCommentNodes(node) - firstDecl = false - else: - commentNodes = gState.getCommentNodes(node[i]) gState.addProc(node[i], node[start], commentNodes) else: # Regular var - discard + gState.addVar(node, i, commentNodes) proc addDef(gState: State, node: TSNode) = # Wrap static inline definition if {.header.} mode is specified diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b2b8c2c..0fea184 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -600,7 +600,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - decho "NODE RESULT: ", result + if result.kind != nkNone: + decho "NODE RESULT: ", result proc parseCExpression*(gState: State, codeRoot: TSNode): PNode = ## Parse a c expression from a root ts node diff --git a/nimterop/getters.nim b/nimterop/getters.nim index e600b18..825b1c4 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -116,7 +116,7 @@ proc getType*(str: string): string = if str == "void": return "object" - result = str.strip(chars={'_'}).replace(re"\s+", " ") + result = str.strip(chars={'_'}).splitWhitespace().join(" ") if gTypeMap.hasKey(result): result = gTypeMap[result] @@ -246,17 +246,23 @@ proc getKeyword*(kind: NimSymKind): string = proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) -proc getPreprocessor*(gState: State, fullpath: string): string = +proc getPreprocessor*(gState: State, fullpath: string) = var cmts = if gState.noComments: "" else: "-CC" cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - rdata: seq[string] = @[] + ddata: seq[string] + rdata: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) + sfileName = sfile.extractFilename() + pDir = sfile.expandFilename().parentDir() + includeDirs: seq[string] + for inc in gState.includeDirs: cmd &= &"-I{inc.sanitizePath} " + includeDirs.add inc.absolutePath().sanitizePath(noQuote = true) for def in gState.defines: cmd &= &"-D{def} " @@ -267,7 +273,10 @@ proc getPreprocessor*(gState: State, fullpath: string): string = else: cmd &= "-D__attribute__(x)= " - cmd &= "-D__restrict= -D__extension__= " + cmd &= "-D__restrict= -D__extension__= -D__inline__=inline -D__inline=inline " + + # https://github.com/tree-sitter/tree-sitter-c/issues/43 + cmd &= "-D_Noreturn= " cmd &= &"{fullpath.sanitizePath}" @@ -278,26 +287,26 @@ proc getPreprocessor*(gState: State, fullpath: string): string = start = false let saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine: - start = true - elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + if sfile in saniLine or + (DirSep notin saniLine and sfileName in saniLine): start = true elif gState.recurse: - let - pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) if pDir.Bl or pDir in saniLine: start = true else: - for inc in gState.includeDirs: - if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: + for inc in includeDirs: + if inc in saniLine: start = true break else: if start: if "#undef" in line: continue - rdata.add line - return rdata.join("\n") + elif line.startsWith("#define"): + ddata.add line + else: + rdata.add line + gState.code = ddata.join("\n") & "\n" & rdata.join("\n") converter toString*(kind: Kind): string = return case kind: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 05284a9..79b1637 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -139,7 +139,7 @@ template Bl*(s: typed): untyped {.used.} = # Redirect output to file when required template gecho*(args: string) = if gState.outputHandle.isNil: - stdout.writeLine(args) + echo args else: gState.outputHandle.writeLine(args) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index f54aa67..02d6494 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -13,7 +13,7 @@ proc process(gState: State, path: string, astTable: AstTable) = gState.mode = getCompilerMode(path) if gState.preprocess: - gState.code = gState.getPreprocessor(path) + gState.getPreprocessor(path) else: gState.code = readFile(path) @@ -35,7 +35,7 @@ proc main( debug = false, defines: seq[string] = @[], dynlib: string = "", - feature: seq[Feature] = @[Feature.ast1], + feature: seq[Feature] = @[], includeDirs: seq[string] = @[], mode = "", nim: string = "nim", @@ -83,6 +83,10 @@ proc main( # Set gDebug in build.nim build.gDebug = debug + # Default `ast` mode + if gState.feature.Bl: + gState.feature.add Feature.ast1 + # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 9565a1f..6760aa9 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -50,7 +50,7 @@ proc getName*(node: TSNode): string {.inline.} = proc getNodeVal*(code: var string, node: TSNode): string = if not node.isNil: - return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1] proc getNodeVal*(gState: State, node: TSNode): string = gState.code.getNodeVal(node) diff --git a/tests/rsa.nim b/tests/rsa.nim index e7bde89..a4aa64b 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -35,10 +35,11 @@ cPlugin: cOverride: proc OPENSSL_die*(assertion: cstring; file: cstring; line: cint) {.importc.} +# Skip comments for https://github.com/tree-sitter/tree-sitter-c/issues/44 cImport(@[ basePath / "rsa.h", basePath / "err.h", -], recurse = true, flags = "-f:ast2 -s " & FLAGS) +], recurse = true, flags = "-f:ast2 -s -c " & FLAGS) {.passL: cryptoLPath.} From 6f96437f39572f3b01f4e61b684f2a74cb524bba Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 18 May 2020 16:39:18 -0500 Subject: [PATCH 155/255] Reduce regex for perf, reduce globals for CT --- nimterop.nimble | 8 +- nimterop/build.nim | 4 +- nimterop/cimport.nim | 8 +- nimterop/getters.nim | 2 +- nimterop/globals.nim | 200 ++++++++++++++++++++++--------------------- nimterop/toast.nim | 2 - nimterop/tshelp.nim | 17 ++++ tests/tast2.nim | 2 + 8 files changed, 128 insertions(+), 115 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 187b0ce..1eb605d 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -30,15 +30,15 @@ proc execTest(test: string, flags = "", runDocs = true) = mkDir docPath buildDocs(@[test], docPath, nimArgs = "--hints:off " & flags) -task buildToast, "build toast": - execCmd("nim c --hints:off nimterop/toast.nim") - task buildTimeit, "build timer": exec "nim c --hints:off -d:danger tests/timeit" -task bt, "build toast": +task buildToast, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") +task bt, "build toast": + buildToastTask() + task btd, "build toast": execCmd("nim c -g nimterop/toast.nim") diff --git a/nimterop/build.nim b/nimterop/build.nim index b02132d..6f13ac7 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep -import regex - type BuildType* = enum btAutoconf, btCmake @@ -995,7 +993,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## prior to the build process. var origname = header.extractFilename().split(".")[0] - name = origname.replace(re"[[:^alnum:]]", "") + name = origname.split(seps = AllChars-Letters-Digits).join() # -d:xxx for this header stdStr = name & "Std" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 99be1f5..0559db7 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,8 +17,6 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import regex - import "."/[build, globals, paths, types] export types @@ -684,7 +682,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: hash = output.hash().abs() hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" npath = hpath[0 .. hpath.rfind('.')] & "nim" - header = ("header" & fullpath.splitFile().name.replace(re"[-.]+", "")) + header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(hpath) or gStateCT.nocache or compileOption("forceBuild"): mkDir(hpath.parentDir()) @@ -715,10 +713,6 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: var nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) - nimout = nimout. - replace(re"([u]?int[\d]+)_t", "$1"). - replace(re"([u]?int)ptr_t", "ptr $1") - if gStateCT.debug: echo nimout diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 825b1c4..716c577 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -227,7 +227,7 @@ proc getOverride*(gState: State, name: string, kind: NimSymKind): string = result = sym.override if kind != nskProc: - result = result.replace(re"(?m)^(.*?)$", " $1") + result = " " & result.replace("\n", "\n ") proc getOverrideFinal*(gState: State, kind: NimSymKind): string = # Get all unused cOverride symbols of `kind` diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 79b1637..ac25a40 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,58 +1,19 @@ -import sequtils, sets, tables, strutils - -import regex - -import "."/plugin +import tables when defined(TOAST): + import sets, sequtils, strutils + + import regex + + import "."/plugin + import compiler/[ast, idents, modulegraphs, options] import "."/treesitter/api -const - gAtoms* {.used.} = @[ - "field_identifier", - "identifier", - "number_literal", - "char_literal", - "preproc_arg", - "primitive_type", - "sized_type_specifier", - "type_identifier" - ].toHashSet() - - gExpressions* {.used.} = @[ - "parenthesized_expression", - "bitwise_expression", - "shift_expression", - "math_expression", - "escape_sequence" - ].toHashSet() - - gEnumVals* {.used.} = @[ - "identifier", - "number_literal", - "char_literal" - ].concat(toSeq(gExpressions.items)) - type - Kind* = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast* = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - when defined(TOAST): - tonim*: proc (ast: ref Ast, node: TSNode, gState: State) - regex*: Regex - - AstTable* {.used.} = TableRef[string, seq[ref Ast]] + Feature* = enum + ast1, ast2 State* = ref object # Command line arguments to toast - some forwarded from cimport.nim @@ -79,31 +40,25 @@ type typeMap*: TableRef[string, string] # `--typeMap | -T` to map instances of type X to Y - e.g. ABC=cint - # cimport.nim specific - compile*: seq[string] # `cCompile()` list of files already processed - nocache*: bool # `cDisableCaching()` to disable caching of artifacts - overrides*: string # `cOverride()` code which gets added to `cPlugin()` output - pluginSource*: string # `cPlugin()` generated code to write to plugin file from - searchDirs*: seq[string] # `cSearchPath()` added directories for header search - - # Data fields - code*: string # Contents of header file currently being processed - currentHeader*: string # Const name of header being currently processed - impShort*: string # Short base name for pragma in output - outputHandle*: File # `--output | -o` open file handle - sourceFile*: string # Full path of header being currently processed - - # Plugin callbacks - onSymbol*, onSymbolOverride*: OnSymbol - onSymbolOverrideFinal*: OnSymbolOverrideFinal - - # Symbol tables - constIdentifiers*: HashSet[string] # Const names for enum casting - identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName - skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or - # the user provided override is blank - # Nim compiler objects when defined(TOAST): + # Data fields + code*: string # Contents of header file currently being processed + currentHeader*: string # Const name of header being currently processed + impShort*: string # Short base name for pragma in output + outputHandle*: File # `--output | -o` open file handle + sourceFile*: string # Full path of header being currently processed + + # Plugin callbacks + onSymbol*, onSymbolOverride*: OnSymbol + onSymbolOverrideFinal*: OnSymbolOverrideFinal + + # Symbol tables + constIdentifiers*: HashSet[string] # Const names for enum casting + identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName + skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or + # the user provided override is blank + + # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode identCache*: IdentCache config*: ConfigRef @@ -112,37 +67,86 @@ type # Table of symbols to generated AST PNode - used to implement forward declarations identifierNodes*: TableRef[string, PNode] - # Used for the exprparser.nim module - currentExpr*, currentTyCastName*: string - # Controls whether or not the current expression - # should validate idents against currently defined idents - skipIdentValidation*: bool + # Used for the exprparser.nim module + currentExpr*, currentTyCastName*: string + # Controls whether or not the current expression + # should validate idents against currently defined idents + skipIdentValidation*: bool - # Legacy AST fields, remove when ast2 becomes default - constStr*, enumStr*, procStr*, typeStr*: string - commentStr*, debugStr*, skipStr*: string - data*: seq[tuple[name, val: string]] - nodeBranch*: seq[string] + # Legacy AST fields, remove when ast2 becomes default + constStr*, enumStr*, procStr*, typeStr*: string + commentStr*, debugStr*, skipStr*: string + data*: seq[tuple[name, val: string]] + nodeBranch*: seq[string] + else: + # cimport.nim specific + compile*: seq[string] # `cCompile()` list of files already processed + nocache*: bool # `cDisableCaching()` to disable caching of artifacts + overrides*: string # `cOverride()` code which gets added to `cPlugin()` output + pluginSource*: string # `cPlugin()` generated code to write to plugin file from + searchDirs*: seq[string] # `cSearchPath()` added directories for header search - Feature* = enum - ast1, ast2 +when defined(TOAST): + const + gAtoms* {.used.} = @[ + "field_identifier", + "identifier", + "number_literal", + "char_literal", + "preproc_arg", + "primitive_type", + "sized_type_specifier", + "type_identifier" + ].toHashSet() -var - gStateCT* {.compiletime, used.} = new(State) + gExpressions* {.used.} = @[ + "parenthesized_expression", + "bitwise_expression", + "shift_expression", + "math_expression", + "escape_sequence" + ].toHashSet() + + gEnumVals* {.used.} = @[ + "identifier", + "number_literal", + "char_literal" + ].concat(toSeq(gExpressions.items)) + + type + Kind* = enum + exactlyOne + oneOrMore # + + zeroOrMore # * + zeroOrOne # ? + orWithNext # ! + + Ast* = object + name*: string + kind*: Kind + recursive*: bool + children*: seq[ref Ast] + tonim*: proc (ast: ref Ast, node: TSNode, gState: State) + regex*: Regex + + AstTable* {.used.} = TableRef[string, seq[ref Ast]] + + # Redirect output to file when required + template gecho*(args: string) = + if gState.outputHandle.isNil: + echo args + else: + gState.outputHandle.writeLine(args) + + template decho*(args: varargs[string, `$`]): untyped = + if gState.debug: + gecho join(args, "").getCommented() +else: + var + gStateCT* {.compiletime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) template Bl*(s: typed): untyped {.used.} = - (s.len == 0) - -# Redirect output to file when required -template gecho*(args: string) = - if gState.outputHandle.isNil: - echo args - else: - gState.outputHandle.writeLine(args) - -template decho*(args: varargs[string, `$`]): untyped = - if gState.debug: - gecho join(args, "").getCommented() \ No newline at end of file + (s.len == 0) \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 02d6494..84835a3 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,8 +4,6 @@ import "."/treesitter/[api, c, cpp] import "."/[ast, ast2, build, globals, getters, grammar, tshelp] -{.passC: "-DNIMTEROP".} - proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 6760aa9..fa2a750 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -79,6 +79,23 @@ proc getStartAtom*(node: TSNode): int = else: break +proc getConstQualifier*(gState: State, node: TSNode): bool = + # Check if node siblings have type_qualifier = `const` + var + curr = node.tsNodePrevNamedSibling() + while not curr.isNil: + # Check previous siblings + if curr.getName() == "type_qualifier" and + gState.getNodeVal(curr) == "const": + return true + curr = curr.tsNodePrevNamedSibling() + + # Check immediate next sibling + curr = node.tsNodePrevNamedSibling() + if curr.getName() == "type_qualifier" and + gState.getNodeVal(curr) == "const": + return true + proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if not node.isNil: # Get number of ntype nodes nested in tree diff --git a/tests/tast2.nim b/tests/tast2.nim index 8deb99e..fd0175a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -2,6 +2,8 @@ import macros, os, sets, strutils import nimterop/[cimport] +{.passC: "-DNIMTEROP".} + static: # Skip casting on lower nim compilers because # the VM does not support it From f079d851db1db0dd64d7538a8b417b1e5c0985a1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 19 May 2020 10:27:31 -0500 Subject: [PATCH 156/255] Changelog for variable support --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index ea3f427..e47974d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,7 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 - Nested function pointers - [#155][i155] [#156][i156] - Various enum fixes - [#159][i159] [#171][i171] - Map `int arr[]` to `arr: UncheckedArray[cint]` - [#174][i174] + - Global variables including arrays and procs (since v0.5.4) - `ast2` also includes an advanced expression parser that can reliably handle constructs typically seen with `#define` statements and enumeration values: - Integers + integer like expressions (hex, octal, suffixes) From 7d3f52f1dc0829bc8540a413925147b96b5298fe Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 19 May 2020 10:27:36 -0500 Subject: [PATCH 157/255] v0.5.4 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1eb605d..d1ad7ce 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.3" +version = "0.5.4" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From e32fd4d1d2542a6ffaeb8221a3da7ef136669651 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 15:16:26 -0500 Subject: [PATCH 158/255] ast2 handle tree-sitter errors, multi-reorder --- nimterop/ast2.nim | 100 +++++++++++++++++++++++++------------------ nimterop/getters.nim | 8 +--- nimterop/globals.nim | 3 ++ 3 files changed, 64 insertions(+), 47 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 3d52e02..e2cdf8f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -27,7 +27,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS if not pnode.isNil: result = pnode[0][0] else: - gecho &"\n# $1'{origname}' skipped" % skind + gecho &"# {skind}'{origname}' skipped" gState.skippedSyms.incl origname proc addOverrideFinal(gState: State, kind: NimSymKind) = @@ -391,7 +391,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: gState.identifierNodes[name] = result else: - gecho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) + gecho &"# {getKeyword(kind)} '{origname}' is duplicate, skipped" proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input @@ -682,7 +682,7 @@ proc newProcTy(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = if node.getVarargs(): gState.addPragma(node, result[^1], "varargs") -proc processNode(gState: State, node: TSNode): bool +proc processNode(gState: State, node: TSNode): Status proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object if not node.isNil: @@ -720,9 +720,9 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = $gState.enumSection[^1][0][1] ) else: - (true, "") + (success, "") - if not processed: + if processed != success: return nil # Add nkIdentDefs for each field @@ -1699,35 +1699,52 @@ proc addDef(gState: State, node: TSNode) = if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: - gecho &"\n# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" % + gecho "# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" % gState.getNodeVal(node[start+1].getAtom()) -proc processNode(gState: State, node: TSNode): bool = - result = true +proc processNode(gState: State, node: TSNode): Status = + const + known = ["preproc_def", "type_definition", + "struct_specifier", "union_specifier", "enum_specifier", + "declaration", "function_definition"].toHashSet() - case node.getName() - of "preproc_def": - gState.addConst(node) - of "type_definition": - if node.len > 0 and node[0].getName() == "enum_specifier": - gState.addEnum(node) - elif node.len > 0 and node[0].getName() == "union_specifier": - gState.addType(node, union = true) + result = success + let + name = node.getName() + if name in known: + # Recognized top-level nodes + let + err = node.anyChildInTree("ERROR") + if not err.isNil: + # Bail on errors + gState.printDebug(node) + gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + result = Status.error else: - gState.addType(node) - of "struct_specifier": - gState.addType(node) - of "union_specifier": - gState.addType(node, union = true) - of "enum_specifier": - gState.addEnum(node) - of "declaration": - gState.addDecl(node) - of "function_definition": - gState.addDef(node) + # Process nodes + case name + of "preproc_def": + gState.addConst(node) + of "type_definition": + if node.len > 0 and node[0].getName() == "enum_specifier": + gState.addEnum(node) + elif node.len > 0 and node[0].getName() == "union_specifier": + gState.addType(node, union = true) + else: + gState.addType(node) + of "struct_specifier": + gState.addType(node) + of "union_specifier": + gState.addType(node, union = true) + of "enum_specifier": + gState.addEnum(node) + of "declaration": + gState.addDecl(node) + of "function_definition": + gState.addDef(node) else: - # Unknown - result = false + # Unknown, will check child nodes + result = unknown proc searchTree(gState: State, root: TSNode) = # Search AST generated by tree-sitter for recognized elements @@ -1735,7 +1752,7 @@ proc searchTree(gState: State, root: TSNode) = node = root nextnode: TSNode depth = 0 - processed = false + processed = success while true: if not node.isNil and depth > -1: @@ -1743,7 +1760,7 @@ proc searchTree(gState: State, root: TSNode) = else: break - if not processed and node.len != 0: + if processed == unknown and node.len != 0: nextnode = node[0] depth += 1 else: @@ -1773,27 +1790,28 @@ proc searchTree(gState: State, root: TSNode) = proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # Create shortcut pragmas to reduce clutter var - hdrPragma: PNode - dynPragma: PNode + count = 0 if not gState.noHeader: # {.pragma: impnameHdr, header: "xxx".} - hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) + let + hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) gState.addPragma(root, hdrPragma, "header", newStrNode(nkStrLit, fullpath)) + gState.pragmaSection.add hdrPragma + count += 1 if gState.dynlib.nBl: # {.pragma: impnameDyn, dynlib: libname.} - dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) + let + dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) - - # Add pragma shortcuts to output - if not hdrPragma.isNil: - gState.pragmaSection.add hdrPragma - if not dynPragma.isNil: gState.pragmaSection.add dynPragma + count += 1 # Add `{.experimental: "codeReordering".} for #206 - gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) + if gState.pragmaSection.len == count: + # Only if not already done + gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) proc initNim*(gState: State) = # Initialize for parseNim() one time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 716c577..0906d56 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -251,7 +251,6 @@ proc getPreprocessor*(gState: State, fullpath: string) = cmts = if gState.noComments: "" else: "-CC" cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - ddata: seq[string] rdata: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) @@ -302,11 +301,8 @@ proc getPreprocessor*(gState: State, fullpath: string) = if start: if "#undef" in line: continue - elif line.startsWith("#define"): - ddata.add line - else: - rdata.add line - gState.code = ddata.join("\n") & "\n" & rdata.join("\n") + rdata.add line + gState.code = rdata.join("\n") converter toString*(kind: Kind): string = return case kind: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index ac25a40..8062ac8 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -131,6 +131,9 @@ when defined(TOAST): AstTable* {.used.} = TableRef[string, seq[ref Ast]] + Status* = enum + success, unknown, error + # Redirect output to file when required template gecho*(args: string) = if gState.outputHandle.isNil: From 80cfc99dcd08a145569f28ad6b9999dbd9868bda Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 16:58:52 -0500 Subject: [PATCH 159/255] Fix nimconf path to nim --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 14 +++++++++----- nimterop/getters.nim | 2 +- nimterop/nimconf.nim | 2 +- nimterop/toast.nim | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index e2cdf8f..b604f1b 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -139,7 +139,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gecho &"# const '{origname}' is duplicate, skipped" else: - gecho &"# const '{origname}' has unsupported value '{val}'" + gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = diff --git a/nimterop/build.nim b/nimterop/build.nim index 6f13ac7..58ecbd1 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -15,6 +15,7 @@ type var gDebug* = false gDebugCT* {.compileTime.} = false + gNimExe* = "" proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") @@ -46,6 +47,14 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gNimExe + # Nim cfg file related functionality include "."/nimconf @@ -64,11 +73,6 @@ proc getNimteropCacheDir(): string = # Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" -proc getCurrentNimCompiler*(): string = - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = ""): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 0906d56..b092663 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -454,7 +454,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = outflags = &"--out:\"{pdll}\"" # Compile plugin as library with `markAndSweep` GC - cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" + cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 2a54d7e..33215ef 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -20,7 +20,7 @@ type proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` var - cmd = "nim --hints:off --dump.format:json dump dummy" + cmd = &"{getCurrentNimCompiler()} --hints:off --dump.format:json dump dummy" dump = "" ret = 0 diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 84835a3..4a9553b 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -64,7 +64,7 @@ proc main( feature: feature, includeDirs: includeDirs, mode: mode, - nim: nim, + nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, past: past, @@ -79,7 +79,8 @@ proc main( ) # Set gDebug in build.nim - build.gDebug = debug + build.gDebug = gState.debug + build.gNimExe = gState.nim # Default `ast` mode if gState.feature.Bl: From 6c01f5504771d2a3b013097007f16a5414baa5b7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 21 May 2020 13:51:15 -0500 Subject: [PATCH 160/255] No pointless hints --- nimterop/ast.nim | 2 ++ nimterop/ast2.nim | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 96116c7..78501fd 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -238,6 +238,8 @@ proc printNim*(gState: State) = if gState.procStr.nBl: gecho &"{gState.procStr}\n" + gecho "{.pop.}" + if gState.debug: if gState.debugStr.nBl: gecho gState.debugStr diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index e2cdf8f..f4e15db 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1815,7 +1815,10 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = proc initNim*(gState: State) = # Initialize for parseNim() one time - gecho "import nimterop/types\n" + gecho """import nimterop/types + +{.push hint[ConvFromXtoItselfNotNeeded]: off.} +""" # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() @@ -1867,3 +1870,5 @@ proc printNim*(gState: State) = tree.add gState.procSection gecho tree.renderTree() + + gecho "{.pop.}" \ No newline at end of file From a54991122de7f586f3671316eaf079b25d1eae16 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 16:58:52 -0500 Subject: [PATCH 161/255] Fix nimconf path to nim --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 14 +++++++++----- nimterop/getters.nim | 2 +- nimterop/nimconf.nim | 2 +- nimterop/toast.nim | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index f4e15db..759ddfd 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -139,7 +139,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gecho &"# const '{origname}' is duplicate, skipped" else: - gecho &"# const '{origname}' has unsupported value '{val}'" + gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = diff --git a/nimterop/build.nim b/nimterop/build.nim index 6f13ac7..58ecbd1 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -15,6 +15,7 @@ type var gDebug* = false gDebugCT* {.compileTime.} = false + gNimExe* = "" proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") @@ -46,6 +47,14 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gNimExe + # Nim cfg file related functionality include "."/nimconf @@ -64,11 +73,6 @@ proc getNimteropCacheDir(): string = # Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" -proc getCurrentNimCompiler*(): string = - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = ""): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 0906d56..b092663 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -454,7 +454,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = outflags = &"--out:\"{pdll}\"" # Compile plugin as library with `markAndSweep` GC - cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" + cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 2a54d7e..33215ef 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -20,7 +20,7 @@ type proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` var - cmd = "nim --hints:off --dump.format:json dump dummy" + cmd = &"{getCurrentNimCompiler()} --hints:off --dump.format:json dump dummy" dump = "" ret = 0 diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 84835a3..4a9553b 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -64,7 +64,7 @@ proc main( feature: feature, includeDirs: includeDirs, mode: mode, - nim: nim, + nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, past: past, @@ -79,7 +79,8 @@ proc main( ) # Set gDebug in build.nim - build.gDebug = debug + build.gDebug = gState.debug + build.gNimExe = gState.nim # Default `ast` mode if gState.feature.Bl: From f1b5f5b8905a015c4e220c7689b7a05ac9375eee Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 21 May 2020 14:57:58 -0500 Subject: [PATCH 162/255] v0.5.5 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index d1ad7ce..76f05ea 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.4" +version = "0.5.5" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From ebf9018df85aa3385b19aed6273d80390753fdeb Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 23 May 2020 11:29:59 -0600 Subject: [PATCH 163/255] Try markdown code block --- nimterop/tshelp.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index fa2a750..ad3e89e 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -284,18 +284,17 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = ## Generate a comment from a set of comment nodes. Comment is guaranteed ## to be able to be rendered using nim doc if commentNodes.len > 0: - result = "::" for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).strip() - result = result.multiReplace( + result = "```\n " & result.multiReplace( { "/**": "", "**/": "", "/*": "", "*/": "", "/*": "", "//": "", "\n": "\n ", "`": "" } # need to replace this last otherwise it supercedes other replacements - ).replace(" *", "").strip() + ).replace(" *", "").strip() & "\n```" proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` From 0e3c2d02f9566786d106687239fc39331e72cf41 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 25 May 2020 22:56:32 -0500 Subject: [PATCH 164/255] v0.5.6 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 76f05ea..3c8e74c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.5" +version = "0.5.6" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From e8baeb92294328d9750469f852816c5a47375954 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 27 May 2020 10:25:48 -0500 Subject: [PATCH 165/255] Fix #222 --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 3 +++ tests/include/tast2.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 759ddfd..de3b452 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -608,7 +608,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # Skip qualifiers after type var start = start0 - while start < node.len - 1 and node[start+1].getName() == "type_qualifier": + while start < node.len - 1 and node[start+1].getName() in ["type_qualifier", "comment"]: start += 1 if start == node.len - 1: diff --git a/nimterop/build.nim b/nimterop/build.nim index 58ecbd1..e0d630a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -365,6 +365,9 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") writeFile(sparsefile, plist) + # In case directory has old files from another run + discard execAction(&"cd {outdirQ} && git clean -fxd") + if checkout.len != 0: echo "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 1) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index b8433f9..019f7c4 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -90,7 +90,7 @@ typedef char *(*A11)[3]; typedef struct A0 *A111[12]; typedef int - **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A12)(int, int b, int *c, int *, int /*out*/ *count[4], int (*func)(int, int)), **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); typedef int (*A13)(int, int, void (*func)(void)); From d875252238ceb6a8134b2c9429f8a6c383aff48a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 27 May 2020 11:54:50 -0500 Subject: [PATCH 166/255] v0.5.7 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 3c8e74c..eed70ba 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.6" +version = "0.5.7" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From fa02a6e0bc955042ac867c5a3a462a00d9cfe092 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 17:42:31 -0500 Subject: [PATCH 167/255] Fix #227 - paramCount/Str revert --- nimterop/docs.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 6c2bbfd..5fe87a1 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -2,11 +2,8 @@ import macros, strformat from os import parentDir, getCurrentCompilerExe, DirSep -when defined(nimdoc) or (NimMajor, NimMinor) >= (1, 3): - from os import paramCount, paramStr - when defined(nimdoc): - from os import getCurrentDir + from os import getCurrentDir, paramCount, paramStr proc getNimRootDir(): string = #[ From 2885753dca1acacd3bcebb299fb7fa2aa7a88fe4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 20:42:42 -0500 Subject: [PATCH 168/255] v0.5.8 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index eed70ba..b5fab89 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.7" +version = "0.5.8" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From eb2fd1c1dd40ad5ecb0ee6389f67c7704375ac85 Mon Sep 17 00:00:00 2001 From: Euan Date: Thu, 18 Jun 2020 19:08:29 +0100 Subject: [PATCH 169/255] Fix #229 - FreeBSD build error --- nimterop/toast.nims | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nimterop/toast.nims b/nimterop/toast.nims index 83d5cf4..e9a3c99 100644 --- a/nimterop/toast.nims +++ b/nimterop/toast.nims @@ -1,10 +1,8 @@ import os # Workaround for C++ scanner.cc causing link error with other C obj files -when defined(MacOSX): - switch("clang.linkerexe", "g++") -else: - switch("gcc.linkerexe", "g++") +switch("clang.linkerexe", "clang++") +switch("gcc.linkerexe", "g++") # Workaround for NilAccessError crash on Windows #98 # Could also help for OSX/Linux crash @@ -25,4 +23,4 @@ when not defined(danger): switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) # Define TOAST for globals.nim -switch("define", "TOAST") \ No newline at end of file +switch("define", "TOAST") From 68cb3ab0b449c6155d5a797bec4d715a80423f05 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 15:10:49 -0500 Subject: [PATCH 170/255] v0.5.9 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index b5fab89..1657aed 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.8" +version = "0.5.9" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 290ad2b96733170ad653abd9f3705b5f50c4dcca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 9 Jun 2020 23:21:11 -0500 Subject: [PATCH 171/255] Initial conan.io support --- nimterop/build.nim | 111 ++++++++++++-- nimterop/conan.nim | 373 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 468 insertions(+), 16 deletions(-) create mode 100644 nimterop/conan.nim diff --git a/nimterop/build.nim b/nimterop/build.nim index e0d630a..d5ce59c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -206,6 +206,14 @@ proc rmDir*(dir: string) = ## Remove a directory or pattern at compile time rmFile(dir, dir = true) +proc cleanDir*(dir: string) = + ## Remove all contents of a directory at compile time + for kind, path in walkDir(dir): + if kind == pcDir: + rmDir(path) + else: + rmFile(path) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -281,7 +289,7 @@ proc extractTar*(tarfile, outdir: string) = if name.len != 0: rmFile(outdir / name) -proc downloadUrl*(url, outdir: string) = +proc downloadUrl*(url, outdir: string, quiet = false) = ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory ## ## If an archive file, it is automatically extracted after download. @@ -291,7 +299,8 @@ proc downloadUrl*(url, outdir: string) = archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] if not (ext in archives and fileExists(outdir/file)): - echo "# Downloading " & file + if not quiet: + echo "# Downloading " & file mkDir(outdir) var cmd = findExe("curl") if cmd.len != 0: @@ -304,7 +313,7 @@ proc downloadUrl*(url, outdir: string) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url, (outdir/file).sanitizePath], retry = 1) + discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) if ext == ".zip": extractZip(file, outdir) @@ -732,6 +741,36 @@ proc getGccLibPaths*(mode: string): seq[string] = when defined(osx): result.add "/usr/lib" +proc getGccInfo*(): tuple[arch, os, compiler, version: string] = + let + (outp, _) = execAction(&"{getCompiler()} -v") + for line in outp.splitLines(): + if line.startsWith("Target: "): + result.arch = line.split(' ')[1].split('-')[0] + result.os = + if "linux" in line: + "linux" + elif "android" in line: + "android" + elif "darwin" in line: + "macos" + elif "w64" in line or "mingw" in line: + "windows" + else: + "unknown" + elif " version " in line: + result.version = line.split(" version ")[1].split(' ')[0] + if "clang" in outp: + if result.os == "macos": + result.compiler = "apple-clang" + else: + result.compiler = "clang" + else: + result.compiler = "gcc" + +# Conan support +include conan + proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) @@ -784,6 +823,28 @@ proc getDlPath(header, url, outdir, version: string): string = result = findFile(header, outdir) +proc getConanPath(header, uri, outdir, version: string, shared: bool): string = + var + uri = uri + + if "$#" in uri or "$1" in uri: + doAssert version.len != 0, "Need version for Conan uri" + uri = uri % version + else: + doAssert version.len == 0, "Conan uri does not contain version" + + let + pkg = newConanPackageFromUri(uri, shared) + downloadConan(pkg, outdir) + + result = findFile(header, outdir) + +proc getConanLPath(outdir: string): string = + let + pkg = loadConanInfo(outdir) + + result = pkg.getConanLibs(outdir).join(" ") + proc getLocalPath(header, outdir: string): string = if outdir.len != 0: result = findFile(header, outdir) @@ -937,7 +998,9 @@ macro isDefined*(def: untyped): untyped = false ) -macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", +macro getHeader*( + header: static[string], giturl: static[string] = "", dlurl: static[string] = "", + conanuri: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -950,16 +1013,18 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux ## `-d:xxxGit` - clone source from a git repo specified in `giturl` ## `-d:xxxDL` - download source from `dlurl` and extract if required + ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` + ## typically formatted as `name/version[@user/channel][:bhash]` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git` or `DL`. - ## This allows using a system installed library if available before falling back to manual - ## building. + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or + ## `Conan`. This allows using a system installed library if available before falling back + ## to manual building. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL, it replaces `$1` in the URL defined. + ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. ## ## All defines can also be set in code using `setDefines()`. ## @@ -1006,6 +1071,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdStr = name & "Std" gitStr = name & "Git" dlStr = name & "DL" + conanStr = name & "Conan" staticStr = name & "Static" verStr = name & "SetVer" @@ -1014,6 +1080,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta nameStd = newIdentNode(stdStr) nameGit = newIdentNode(gitStr) nameDL = newIdentNode(dlStr) + nameConan = newIdentNode(conanStr) nameStatic = newIdentNode(staticStr) @@ -1031,6 +1098,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdVal = gDefines.hasKey(stdStr) gitVal = gDefines.hasKey(gitStr) dlVal = gDefines.hasKey(dlStr) + conanVal = gDefines.hasKey(conanStr) staticVal = gDefines.hasKey(staticStr) verVal = if gDefines.hasKey(verStr): @@ -1052,14 +1120,17 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 + `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, outdir, version: string): string = + proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) + elif `nameConan`: + getConanPath(header, conanuri, outdir, version, shared) else: getLocalPath(header, outdir) @@ -1077,23 +1148,30 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdLPath = when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" + useStd = stdPath.len != 0 and stdLPath.len != 0 + # Look elsewhere if requested while prioritizing standard paths prePath = - when stdPath.len != 0 and stdLPath.len != 0: + when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) - # Run preBuild hook before building library if not standard - when (prePath != stdPath or prePath.len == 0) and declared(`preBuild`): + # Run preBuild hook before building library if not Std or Conan + when not (useStd or `nameConan`) and declared(`preBuild`): static: `preBuild`(`outdir`, prePath) const # Library binary path - build if not standard `lpath`* = - when stdPath.len != 0 and stdLPath.len != 0: + when useStd: stdLPath + elif `nameConan`: + when `nameStatic`: + getConanLPath(`outdir`) + else: + findFile(`lname`, `outdir`, regex = true) else: buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) @@ -1102,10 +1180,11 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) static: - doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name` + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name` doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` diff --git a/nimterop/conan.nim b/nimterop/conan.nim new file mode 100644 index 0000000..0707ecc --- /dev/null +++ b/nimterop/conan.nim @@ -0,0 +1,373 @@ +import json, marshal, os, strformat, strutils, tables + +type + ConanPackage* = ref object + ## ConanPackage type that stores conan uri and recipes/builds/revisions + name*: string + version*: string + user*: string + channel*: string + recipes*: OrderedTableRef[string, seq[ConanBuild]] + + bhash*: string + shared*: bool + headers*: seq[string] + libs*: seq[string] + requires*: seq[ConanPackage] + + ConanBuild* = ref object + ## Build type that stores build specific info and revisions + bhash*: string + settings*: TableRef[string, string] + options*: TableRef[string, string] + requires*: seq[string] + recipe_hash*: string + revisions*: seq[string] + +const + # Conan API urls + baseUrl = "https://conan.bintray.com/v2/conans" + searchUrl = baseUrl & "/search?q=$query" + pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query" + cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" + dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" + + # Bintray download sub-URL for explicit `user/channel` (not _/_) + dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" + + # Strings + conanInfo = "conaninfo.json" + conanPackage = "conan_package.tgz" + conanManifest = "conanmanifest.txt" + +# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim +# https://bintray.com/conan/conan-center +# https://bintray.com/bincrafters/public-conan/ + +# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared + +var + # Bintray download URL for explicit `user/channel` + baseAltUrl {.compiletime.} = { + "bincrafters": "https://bintray.com/bincrafters/public-conan", + "conan": "https://bintray.com/conan-community/conan" + }.toTable() + +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + +proc addAltBaseUrl*(name, url: string) = + # Add an alternate base URL for a custom conan repo on bintray + baseAltUrl[name] = url + +proc jsonGet(url: string): JsonNode = + # Make HTTP call and return content as JSON + let + temp = getTempDir() + file = temp / url.extractFilename() + + downloadUrl(url, temp, quiet = true) + result = readFile(file).parseJson() + rmFile(file) + +proc `==`(pkg1, pkg2: ConanPackage): bool = + (not pkg1.isNil and not pkg2.isNil and + pkg1.name == pkg2.name and + pkg1.version == pkg2.version and + pkg1.user == pkg2.user and + pkg1.channel == pkg2.channel and + + pkg1.bhash == pkg2.bhash and + pkg1.shared == pkg2.shared) + +proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shared = true): ConanPackage = + ## Create a new ConanPackage with specified name and version + result = new(ConanPackage) + result.name = name + result.version = version + result.user = user + result.channel = channel + result.recipes = newOrderedTable[string, seq[ConanBuild]](2) + + result.bhash = bhash + result.shared = shared + +proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = + ## Create a new ConanPackage from a conan uri + ## + ## name/version[@user/channel][:bhash] + var + name, version, user, channel, bhash: string + + spl = uri.split(":") + + if spl.len > 1: + bhash = spl[1] + + spl = spl[0].split('/') + + name = spl[0] + user = "_" + channel = "_" + + if spl.len > 2: + channel = spl[2] + if spl.len > 1: + spl = spl[1].split('@') + + version = spl[0] + if spl.len > 1: + user = spl[1] + + result = newConanPackage(name, version, user, channel, bhash, shared) + +proc getUriFromConanPackage*(pkg: ConanPackage): string = + ## Convert a ConanPackage to a conan uri + result = pkg.name + if pkg.version.len != 0: + result &= "/" & pkg.version + if pkg.user.len != 0: + result &= "@" & pkg.user + if pkg.channel.len != 0: + result &= "/" & pkg.channel + if pkg.bhash.len != 0: + result &= ":" & pkg.bhash + +proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = + ## Search for package by `name` and optional `version`, `user` and `channel` + var + query = name + if version.len != 0: + query &= "/" & version + if user.len != 0: + query &= "@" & user + if channel.len != 0: + query &= "/" & channel + + let + j1 = jsonGet(searchUrl % ["query", query]) + res = j1.getOrDefault("results").getElems() + + if res.len != 0: + # Return last entry - latest + result = newConanPackageFromUri(res[^1].getStr()) + +proc searchConan*(pkg: ConanPackage): ConanPackage = + ## Search for latest package based on incomplete package info + result = searchConan(pkg.name, pkg.version, pkg.user, pkg.channel) + +proc getConanBuilds*(pkg: ConanPackage, filter = "") = + ## Get all builds for a package based on the C compiler's target OS/arch info + ## + ## `filter` can be used to tweak search terms + ## e.g. build_type=Debug&compiler=clang + let + (arch, os, compiler, _) = getGccInfo() + + query = + if pkg.bhash.len == 0: + block: + var + query = &"?q=arch={arch}&os={os.capitalizeAscii()}" + if "build_type" notin filter: + query &= "&build_type=Release" + if "shared=" notin filter: + query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" + if filter.len != 0: + query &= &"&{filter}" + if "compiler=" notin filter and os != "windows": + query &= &"&compiler={compiler}" + query.replace("&", "%20and%20") + else: "" + + url = pkgUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "query", query + ] + + j1 = jsonGet(url) + + if not j1.isNil: + for bhash, bdata in j1.getFields(): + if pkg.bhash.len == 0 or pkg.bhash == bhash: + let + bld = new(ConanBuild) + settings = bdata.getOrDefault("settings") + options = bdata.getOrDefault("options") + requires = bdata.getOrDefault("requires") + bld.bhash = bhash + if not settings.isNil: + bld.settings = newTable[string, string](8) + for key, value in settings.getFields(): + bld.settings[key] = value.getStr() + if not options.isNil: + bld.options = newTable[string, string](8) + for key, value in options.getFields(): + bld.options[key] = value.getStr() + bld.requires = requires.to(seq[string]) + bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr() + + if pkg.recipes.hasKey(bld.recipe_hash): + pkg.recipes[bld.recipe_hash].add bld + else: + pkg.recipes[bld.recipe_hash] = @[bld] + + # Only need first or matching build + break + +proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = + ## Get all revisions of a build + let + url = cfgUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "recipe", bld.recipe_hash, + "build", bld.bhash + ] + + j1 = jsonGet(url) + + if not j1.isNil: + let + revs = j1.getOrDefault("revisions") + for i in revs: + bld.revisions.add i.getOrDefault("revision").getStr() + +proc loadConanInfo*(outdir: string): ConanPackage = + ## Load cached package info from `outdir/conaninfo.txt` + fixOutDir() + let + file = outdir / conanInfo + + if fileExists(file): + result = to[ConanPackage](readFile(file)) + +proc saveConanInfo*(pkg: ConanPackage, outdir: string) = + ## Save downloaded package info to `outdir/conaninfo.txt` + fixOutDir() + let + file = outdir / conanInfo + + writeFile(file, $$pkg) + +proc parseConanManifest(pkg: ConanPackage, outdir: string) = + let + file = outdir / conanManifest + + if fileExists(file): + let + data = readFile(file) + for line in data.splitLines(): + let + line = line.split(':')[0] + if line.startsWith("include/"): + pkg.headers.add line + elif line.startsWith("lib/"): + pkg.libs.add line + elif line.startsWith("bin/") and line.endsWith("dll"): + pkg.libs.add line + +proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") = + ## Download specific `revision` of `bld` to `outdir` + ## + ## If omitted, the latest revision (first) is downloaded + fixOutDir() + let + revision = + if revision.len != 0: + revision + else: + bld.revisions[0] + + url = + if pkg.user == "_": + dlUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "recipe", bld.recipe_hash, + "build", bld.bhash, + "revision", revision, + "file", conanPackage + ] + else: + baseAltUrl[pkg.user] & dlAltUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "build", bld.bhash, + "file", conanPackage + ] + + downloadUrl(url, outdir, quiet = true) + downloadUrl(url.replace(conanPackage, conanManifest), outdir, quiet = true) + + pkg.parseConanManifest(outdir) + + rmFile(outdir / url.extractFilename()) + rmFile(outdir / conanManifest) + +proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) +proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = + ## Download latest recipe/build/revision of `pkg` to `outdir` + fixOutDir() + let + pkg = + if pkg.version.len == 0: + searchConan(pkg) + else: + pkg + + cpkg = loadConanInfo(outdir) + + if cpkg == pkg: + return + elif clean: + cleanDir(outdir) + + pkg.getConanBuilds() + + for recipe, builds in pkg.recipes: + for build in builds: + if pkg.bhash.len == 0 or pkg.bhash == build.bhash: + pkg.getConanRevisions(build) + pkg.dlConanBuild(build, outdir) + pkg.dlConanRequires(build, outdir) + break + break + + if clean: + pkg.saveConanInfo(outdir) + +proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = + ## Download all required dependencies of this `bld` + ## + ## This is not required for shared libs since conan builds them + ## with all dependencies statically linked in + fixOutDir() + if bld.options["shared"] == "False": + for req in bld.requires: + let + rpkg = newConanPackageFromUri(req) + + downloadConan(rpkg, outdir, clean = false) + pkg.requires.add rpkg + +proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = + ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps + ## in descending order + ## + ## `outdir` is prefixed to each entry + for lib in pkg.libs: + result.add outdir / lib + + for cpkg in pkg.requires: + result.add cpkg.getConanLibs(outdir) \ No newline at end of file From 6d256d127191e546b51c5707ae1bfebac43486c6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 12 Jun 2020 17:45:06 -0500 Subject: [PATCH 172/255] Filter compiler version, Windows .lib detection --- nimterop/build.nim | 2 +- nimterop/conan.nim | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index d5ce59c..fd7ad3a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1138,7 +1138,7 @@ macro getHeader*( `version`* {.strdefine.} = `verVal` `lname` = when `nameStatic`: - `lre` & ".a" + `lre` & "\\.(a|lib)" else: `lre` & getDynlibExt() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 0707ecc..1a9237a 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -163,7 +163,15 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = ## `filter` can be used to tweak search terms ## e.g. build_type=Debug&compiler=clang let - (arch, os, compiler, _) = getGccInfo() + (arch, os, compiler, version) = getGccInfo() + + vsplit = version.split('.') + + vfilter = + when defined(OSX): + vsplit[0 .. 1].join(".") + else: + vsplit[0] query = if pkg.bhash.len == 0: @@ -177,7 +185,8 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if filter.len != 0: query &= &"&{filter}" if "compiler=" notin filter and os != "windows": - query &= &"&compiler={compiler}" + query &= &"&compiler={compiler}&compiler.version=" & vfilter + query.replace("&", "%20and%20") else: "" @@ -370,4 +379,4 @@ proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = result.add outdir / lib for cpkg in pkg.requires: - result.add cpkg.getConanLibs(outdir) \ No newline at end of file + result.add cpkg.getConanLibs(outdir) From c63d82bf77ca6245469586265950f6e2085546f0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 15:16:21 -0500 Subject: [PATCH 173/255] Add tests for conan, recurse implies preprocess --- nimterop/build.nim | 25 +++++++++++++++--------- nimterop/conan.nim | 2 ++ nimterop/toast.nim | 6 +++++- tests/getheader.nims | 8 ++++++++ tests/libssh2.nim | 46 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 tests/libssh2.nim diff --git a/nimterop/build.nim b/nimterop/build.nim index fd7ad3a..d534cf9 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1018,26 +1018,33 @@ macro getHeader*( ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. + ## The user can opt to download the sources to `outdir` using any other method such as + ## git sub-modules, vendoring or pointing to a repository that was already cloned. ## ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or ## `Conan`. This allows using a system installed library if available before falling back - ## to manual building. + ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other + ## methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. ## - ## All defines can also be set in code using `setDefines()`. + ## All defines can also be set in code using `setDefines()` and checked for using + ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. ## ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package - ## manager was used to install prebuilt headers and binaries. + ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` + ## which downloads pre-built binaries. ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. + ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will + ## include all separated by space in the right order for linking. ## ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, ## `cmake` and `make` in case additional configuration is required as part of the build process. @@ -1047,19 +1054,19 @@ macro getHeader*( ## with cmake. In this case, `altNames = "z,zlib"`. Comma separate for multiple alternate names without ## spaces. ## - ## `buildTypes` specifies a list of in order build strategies to use when building the downloaded source - ## files. Default is [btCmake, btAutoconf] - ## ## The original header name is not included by default if `altNames` is set since it could cause the ## wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if `altNames = "SDL2"`. ## Explicitly include it in `altNames` like the `zlib` example when required. ## + ## `buildTypes` specifies a list of ordered build strategies to use when building the downloaded source + ## files. Default is [btCmake, btAutoconf] + ## ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or downloaded but ## before the library is built. This might be needed if some initial prep needs to be done before ## compilation. A few values are provided to the hook to help provide context: ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the - ## downloaded source code. + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the + ## downloaded source code. ## ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called ## prior to the build process. diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 1a9237a..e9d23cb 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -342,6 +342,8 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + pkg.getConanBuilds() for recipe, builds in pkg.recipes: diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 4a9553b..1844a56 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -123,6 +123,10 @@ proc main( if check and outputFile.len == 0: outputFile = getTempDir() / "toast_" & ($getTime().toUnix()).addFileExt("nim") + # Recurse implies preprocess + if gState.recurse: + gState.preprocess = true + # Redirect output to file if outputFile.len != 0: doAssert gState.outputHandle.open(outputFile, fmWrite), @@ -248,7 +252,7 @@ when isMainModule: "pnim": "print Nim output", "prefix": "strip prefix from identifiers", "preprocess": "run preprocessor on header", - "recurse": "process #include files", + "recurse": "process #include files - implies --preprocess", "replace": "replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex", "source" : "C/C++ source/header(s) and command line file(s)", "stub": "stub out undefined type references as objects", diff --git a/tests/getheader.nims b/tests/getheader.nims index 5d3b427..55d00f1 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -17,10 +17,12 @@ var cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\" -d:checkAbi" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" + sshcmd = " -r libssh2.nim" lexp = "liblzma version = " zexp = "zlib version = " testCall(cmd & lrcmd, "No build files found", 1) +testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) when defined(posix): # stdlib @@ -35,6 +37,9 @@ when defined(posix): testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) + # conan static + testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) @@ -51,3 +56,6 @@ testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5 # dl testCall(cmd & " -d:zlibDL -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0) testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0, delete = false) + +# conan +testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) diff --git a/tests/libssh2.nim b/tests/libssh2.nim new file mode 100644 index 0000000..b7e591c --- /dev/null +++ b/tests/libssh2.nim @@ -0,0 +1,46 @@ +import nimterop/[build, cimport] + +const + outdir = getProjectCacheDir("libssh2") + +getHeader( + header = "libssh2.h", + conanuri = "libssh2/$1", + outdir = outdir +) + +cOverride: + type + stat = object + stat64 = object + SOCKET = object + +when not libssh2Static: + cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") + + when not defined(Windows): + proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} +else: + cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") + + when not defined(Windows): + proc zlibVersion(): cstring {.importc.} + + {.passL: "-lpthread".} + +assert libssh2_init(0) == 0 + +let + session = libssh2_session_init_ex(nil, nil, nil, nil) + +if session == nil: + quit(1) + +libssh2_session_set_blocking(session, 0.cint) + +echo "zlib version = " & (block: + when not defined(Windows): + $zlibVersion() + else: + "" +) \ No newline at end of file From b6278280dcc5018f043a1112c003e31c8305edab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 16:10:34 -0500 Subject: [PATCH 174/255] Fix for Windows, README --- README.md | 11 ++++++----- nimterop/build.nim | 45 +++++++++++++++++++++++++++------------------ nimterop/conan.nim | 24 +++++++++++++++++------- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 48822ac..3fd48e2 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ getHeader( "header.h", # The header file to wrap, full path is returned in `headerPath` giturl = "https://github.com/username/repo", # Git repo URL dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file + conanuri = "repo/$1", # Conan.io URI outdir = baseDir, # Where to download/build/search conFlags = "--disable-comp --enable-feature", # Flags to pass configure script cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake @@ -61,7 +62,7 @@ getHeader( # Wrap headerPath as returned from getHeader() and link statically # or dynamically depending on user input -when not defined(headerStatic): +when not isDefined(headerStatic): cImport(headerPath, recurse = true, dynlib = "headerLPath") # Pass dynlib if not static link else: cImport(headerPath, recurse = true) @@ -74,8 +75,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. -- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. - - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL if defined. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. #### Pre build @@ -92,7 +93,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde #### Linking -- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If it is omitted, the dynamic library is returned in `headerLPath`. +- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. - `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux - `libheader.dylib` on OSX @@ -224,7 +225,7 @@ Options: -n, --pnim bool false print Nim output -E=, --prefix= strings {} strip prefix from identifiers -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files + -r, --recurse bool false process #include files, implies --preprocess -G=, --replace= strings {} replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers diff --git a/nimterop/build.nim b/nimterop/build.nim index d534cf9..74e6a61 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1044,32 +1044,41 @@ macro getHeader*( ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will - ## include all separated by space in the right order for linking. + ## include paths to all of them separated by space in the right order for linking. + ## + ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of + ## OSX will complain if statically linking to these binaries. Further, all Conan binaries + ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc + ## or clang might lead to incompatibility issues if the library uses Visual Studio + ## specific compiler features. ## ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, - ## `cmake` and `make` in case additional configuration is required as part of the build process. + ## `cmake` and `make` in case additional configuration is required as part of the build + ## process. ## - ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for the header but - ## the typical lib name is `libz.so` and not `libzlib.so`. However, it is libzlib.dll on Windows if built - ## with cmake. In this case, `altNames = "z,zlib"`. Comma separate for multiple alternate names without - ## spaces. + ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for + ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is + ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma + ## separate for multiple alternate names without spaces. ## - ## The original header name is not included by default if `altNames` is set since it could cause the - ## wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if `altNames = "SDL2"`. - ## Explicitly include it in `altNames` like the `zlib` example when required. + ## The original header name is not included by default if `altNames` is set since it could + ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if + ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when + ## required. ## - ## `buildTypes` specifies a list of ordered build strategies to use when building the downloaded source - ## files. Default is [btCmake, btAutoconf] + ## `buildTypes` specifies a list of ordered build strategies to use when building the + ## downloaded source files. Default is [btCmake, btAutoconf] ## - ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or downloaded but - ## before the library is built. This might be needed if some initial prep needs to be done before - ## compilation. A few values are provided to the hook to help provide context: + ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or + ## downloaded but before the library is built. This might be needed if some initial prep + ## needs to be done before compilation. A few values are provided to the hook to help + ## provide context: ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the - ## downloaded source code. + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path + ## in the downloaded source code. ## - ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called - ## prior to the build process. + ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get + ## called prior to the build process. var origname = header.extractFilename().split(".")[0] name = origname.split(seps = AllChars-Letters-Digits).join() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index e9d23cb..872d138 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -65,13 +65,19 @@ proc jsonGet(url: string): JsonNode = # Make HTTP call and return content as JSON let temp = getTempDir() - file = temp / url.extractFilename() + file = block: + var + file = temp / url.extractFilename() + when defined(Windows): + file = file.replace('?', '_') + file downloadUrl(url, temp, quiet = true) result = readFile(file).parseJson() rmFile(file) -proc `==`(pkg1, pkg2: ConanPackage): bool = +proc `==`*(pkg1, pkg2: ConanPackage): bool = + ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and pkg1.name == pkg2.name and pkg1.version == pkg2.version and @@ -94,9 +100,7 @@ proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shar result.shared = shared proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = - ## Create a new ConanPackage from a conan uri - ## - ## name/version[@user/channel][:bhash] + ## Create a new ConanPackage from a conan uri typically formatted as name/version[@user/channel][:bhash] var name, version, user, channel, bhash: string @@ -136,6 +140,8 @@ proc getUriFromConanPackage*(pkg: ConanPackage): string = proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = ## Search for package by `name` and optional `version`, `user` and `channel` + ## + ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name if version.len != 0: @@ -249,7 +255,7 @@ proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = bld.revisions.add i.getOrDefault("revision").getStr() proc loadConanInfo*(outdir: string): ConanPackage = - ## Load cached package info from `outdir/conaninfo.txt` + ## Load cached package info from `outdir/conaninfo.json` fixOutDir() let file = outdir / conanInfo @@ -258,7 +264,7 @@ proc loadConanInfo*(outdir: string): ConanPackage = result = to[ConanPackage](readFile(file)) proc saveConanInfo*(pkg: ConanPackage, outdir: string) = - ## Save downloaded package info to `outdir/conaninfo.txt` + ## Save downloaded package info to `outdir/conaninfo.json` fixOutDir() let file = outdir / conanInfo @@ -266,6 +272,7 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = writeFile(file, $$pkg) proc parseConanManifest(pkg: ConanPackage, outdir: string) = + # Get all header and library info from downloaded conan package let file = outdir / conanManifest @@ -327,6 +334,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` + ## + ## High-level API that handles the end to end Conan process flow to find + ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() let pkg = From f21dab89caab28c20fa7db65969c8089fece55c9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 16:53:24 -0500 Subject: [PATCH 175/255] Handle bad conan uri, test Windows static --- nimterop/conan.nim | 16 +++++++--------- tests/getheader.nims | 3 +++ tests/zlib.nim | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 872d138..95ab24f 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -40,12 +40,6 @@ const conanPackage = "conan_package.tgz" conanManifest = "conanmanifest.txt" -# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim -# https://bintray.com/conan/conan-center -# https://bintray.com/bincrafters/public-conan/ - -# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared - var # Bintray download URL for explicit `user/channel` baseAltUrl {.compiletime.} = { @@ -73,7 +67,10 @@ proc jsonGet(url: string): JsonNode = file downloadUrl(url, temp, quiet = true) - result = readFile(file).parseJson() + try: + result = readFile(file).parseJson() + except JsonParsingError: + discard rmFile(file) proc `==`*(pkg1, pkg2: ConanPackage): bool = @@ -352,10 +349,11 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" - pkg.getConanBuilds() + doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: diff --git a/tests/getheader.nims b/tests/getheader.nims index 55d00f1..349b9bb 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -39,6 +39,9 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) +else: + # conan static for Windows + testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) diff --git a/tests/zlib.nim b/tests/zlib.nim index 852dca9..15400d4 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -27,6 +27,7 @@ getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", + conanuri = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) From 2b3b5e2bec8f7c39540f0da5b4331f5e90c46843 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 15 Jun 2020 21:37:35 -0500 Subject: [PATCH 176/255] Add binarybuilder.org support, conan improvements, headerLDeps --- README.md | 30 ++++--- nimterop/build.nim | 196 ++++++++++++++++++++++++++++++---------- nimterop/conan.nim | 78 ++++++++-------- nimterop/jbb.nim | 210 +++++++++++++++++++++++++++++++++++++++++++ tests/getheader.nims | 6 ++ tests/libssh2.nim | 14 +-- tests/zlib.nim | 4 + 7 files changed, 437 insertions(+), 101 deletions(-) create mode 100644 nimterop/jbb.nim diff --git a/README.md b/README.md index 3fd48e2..ca17b90 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ getHeader( giturl = "https://github.com/username/repo", # Git repo URL dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file conanuri = "repo/$1", # Conan.io URI + jbburi = "repo/$1", # BinaryBuilder.org URI outdir = baseDir, # Where to download/build/search conFlags = "--disable-comp --enable-feature", # Flags to pass configure script cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake @@ -75,8 +76,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. -- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center. - - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL, `-d:headerConan` to download from https://conan.io/center or `-d:headerJBB` to download from https://binarybuilder.org. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL, Conan and JBB, it replaces `$1` in the URL if specified. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. #### Pre build @@ -93,16 +94,21 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde #### Linking -- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -- `getHeader()` searches for libraries based on the header name by default: - - `libheader.so` or `libheader.a` on Linux - - `libheader.dylib` on OSX - - `header.dll` or `header.a` on Windows -- If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. - - For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. - - In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. - - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. -- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. +If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. + +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. + +`getHeader()` searches for libraries based on the header name by default: +- `libheader.so` or `libheader.a` on Linux +- `libheader.dylib` on OSX +- `header.dll`, `header.a` or `header.lib` on Windows + +If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. +- For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. +- In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. +- See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. + +[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. #### User control diff --git a/nimterop/build.nim b/nimterop/build.nim index 74e6a61..eda27e7 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -158,7 +158,7 @@ proc mkDir*(dir: string) = flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) -proc cpFile*(source, dest: string, move=false) = +proc cpFile*(source, dest: string, move = false) = ## Copy a file from `source` to `dest` at compile time let source = source.replace("/", $DirSep) @@ -214,6 +214,25 @@ proc cleanDir*(dir: string) = else: rmFile(path) +proc cpTree*(source, dest: string, move = false) = + ## Copy contents of source dir to the destination, not the directory itself + for kind, path in walkDir(source, relative = true): + if kind == pcDir: + cpTree(source / path, dest / path, move) + if move: + rmDir(source / path) + else: + if not dirExists(dest): + mkDir(dest) + if move: + mvFile(source / path, dest / path) + else: + cpFile(source / path, dest / path) + +proc mvTree*(source, dest: string) = + ## Move contents of source dir to the destination, not the directory itself + cpTree(source, dest, move = true) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -242,7 +261,7 @@ proc getProjectCacheDir*(name: string, forceClean = true): string = echo "# Removing " & result rmDir(result) -proc extractZip*(zipfile, outdir: string) = +proc extractZip*(zipfile, outdir: string, quiet = false) = ## Extract a zip file using `powershell` on Windows and `unzip` on other ## systems to the specified output directory var cmd = "unzip -o $#" @@ -251,10 +270,11 @@ proc extractZip*(zipfile, outdir: string) = "'System.IO.Compression.FileSystem'; " & "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - echo "# Extracting " & zipfile + if not quiet: + echo "# Extracting " & zipfile discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") -proc extractTar*(tarfile, outdir: string) = +proc extractTar*(tarfile, outdir: string, quiet = false) = ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory var cmd = "" @@ -284,7 +304,8 @@ proc extractTar*(tarfile, outdir: string) = doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath - echo "# Extracting " & tarfile + if not quiet: + echo "# Extracting " & tarfile discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -316,9 +337,9 @@ proc downloadUrl*(url, outdir: string, quiet = false) = discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) if ext == ".zip": - extractZip(file, outdir) + extractZip(file, outdir, quiet) elif ext in archives: - extractTar(file, outdir) + extractTar(file, outdir, quiet) proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory @@ -342,7 +363,7 @@ proc gitCheckout*(file, outdir: string) = sleep(500) echo "# Retrying ..." -proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = +proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## ## `plist` is the list of specific files and directories or wildcards @@ -362,7 +383,8 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = mkDir(outdir) - echo "# Setting up Git repo: " & url + if not quiet: + echo "# Setting up Git repo: " & url discard execAction(&"cd {outdirQ} && git init .") discard execAction(&"cd {outdirQ} && git remote add origin {url}") @@ -378,20 +400,32 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = discard execAction(&"cd {outdirQ} && git clean -fxd") if checkout.len != 0: - echo "# Checking out " & checkout + if not quiet: + echo "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 1) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: - echo "# Pulling repository" + if not quiet: + echo "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1) -proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = - ## Find the file in the specified directory +proc gitTags*(outdir: string): seq[string] = + ## Get all the git tags in the specified directory + let + cmd = &"cd {outdir.sanitizePath} && git tag" + tags = execAction(cmd).output.splitLines() + for tag in tags: + let + tag = tag.strip() + if tag.len != 0: + result.add tag + +proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = + ## Find all matching files in the specified directory ## ## `file` is a regular expression if `regex` is true ## - ## Turn off recursive search with `recurse` and stop on first match with - ## `first`. Without it, the shortest match is returned. + ## Turn off recursive search with `recurse` var cmd = when defined(Windows): @@ -435,10 +469,22 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = "" else: line + if f.len != 0: + result.add f - if (f.len != 0 and (result.len == 0 or result.len > f.len)): - result = f - if first: break +proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = + ## Find the file in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. + let + matches = findFiles(file, dir, recurse, regex) + for match in matches: + if (result.len == 0 or result.len > match.len): + result = match + if first: break proc flagBuild*(base: string, flags: openArray[string]): string = ## Simple helper proc to generate flags for `configure`, `cmake`, etc. @@ -533,7 +579,7 @@ proc configure*(path, check: string, flags = "") = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# Configure failed" + doAssert (path / check).fileExists(), "Configure failed" proc getCmakePropertyStr(name, property, value: string): string = &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" @@ -630,7 +676,7 @@ proc cmake*(path, check, flags: string) = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# cmake failed" + doAssert (path / check).fileExists(), "cmake failed" proc make*(path, check: string, flags = "", regex = false) = ## Run the `make` command to build all binaries in the specified path @@ -666,7 +712,7 @@ proc make*(path, check: string, flags = "", regex = false) = echoDebug execAction(cmd).output - doAssert findFile(check, path, regex = regex).len != 0, "# make failed" + doAssert findFile(check, path, regex = regex).len != 0, "make failed" proc getCompilerMode*(path: string): string = ## Determines a target language mode from an input filename, if one is not already specified. @@ -768,9 +814,16 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] = else: result.compiler = "gcc" +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + # Conan support include conan +# Julia Binary Builder support +include jbb + proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) @@ -839,11 +892,41 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = result = findFile(header, outdir) -proc getConanLPath(outdir: string): string = +proc getConanLDeps(outdir: string): seq[string] = let pkg = loadConanInfo(outdir) - - result = pkg.getConanLibs(outdir).join(" ") + + result = pkg.getConanLDeps(outdir) + +proc getJBBPath(header, uri, outdir, version: string): string = + let + spl = uri.split('/', 1) + name = spl[0] + + var + ver = + if spl.len == 2: + spl[1] + else: + "" + + if "$#" in ver or "$1" in ver: + doAssert version.len != 0, "Need version for BinaryBuilder.org uri" + ver = ver % version + else: + doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + + let + pkg = newJBBPackage(name, ver) + downloadJBB(pkg, outdir) + + result = findFile(header, outdir) + +proc getJBBLDeps(outdir: string, shared: bool): seq[string] = + let + pkg = loadJBBInfo(outdir) + + result = pkg.getJBBLDeps(outdir, shared) proc getLocalPath(header, outdir: string): string = if outdir.len != 0: @@ -932,7 +1015,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT buildStatus.built = true let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir - doAssert buildStatus.built, &"\n# Build configuration failed - {error}\n" + doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) @@ -1000,7 +1083,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*( header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", outdir: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -1015,19 +1098,22 @@ macro getHeader*( ## `-d:xxxDL` - download source from `dlurl` and extract if required ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` ## typically formatted as `name/version[@user/channel][:bhash]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` + ## typically formatted as `name/version` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## The user can opt to download the sources to `outdir` using any other method such as ## git sub-modules, vendoring or pointing to a repository that was already cloned. ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or - ## `Conan`. This allows using a system installed library if available before falling back - ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other - ## methods. + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, + ## `Conan` or `JBB`. This allows using a system installed library if available before + ## falling back to manual building. The user would need to specify both `-d:xxxStd` and + ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. + ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL + ## defined. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1035,7 +1121,7 @@ macro getHeader*( ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` - ## which downloads pre-built binaries. + ## or `-d:xxxJBB` which download pre-built binaries. ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can @@ -1043,9 +1129,10 @@ macro getHeader*( ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note - ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will - ## include paths to all of them separated by space in the right order for linking. - ## + ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the + ## `xxxLPath` will include paths to all of them separated by space in the right order for + ## linking. + ## ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of ## OSX will complain if statically linking to these binaries. Further, all Conan binaries ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc @@ -1088,6 +1175,7 @@ macro getHeader*( gitStr = name & "Git" dlStr = name & "DL" conanStr = name & "Conan" + jbbStr = name & "JBB" staticStr = name & "Static" verStr = name & "SetVer" @@ -1097,12 +1185,14 @@ macro getHeader*( nameGit = newIdentNode(gitStr) nameDL = newIdentNode(dlStr) nameConan = newIdentNode(conanStr) + nameJBB = newIdentNode(jbbStr) nameStatic = newIdentNode(staticStr) # Consts to generate path = newIdentNode(name & "Path") lpath = newIdentNode(name & "LPath") + ldeps = newIdentNode(name & "LDeps") version = newIdentNode(verStr) lname = newIdentNode(name & "LName") preBuild = newIdentNode(name & "PreBuild") @@ -1115,6 +1205,7 @@ macro getHeader*( gitVal = gDefines.hasKey(gitStr) dlVal = gDefines.hasKey(dlStr) conanVal = gDefines.hasKey(conanStr) + jbbVal = gDefines.hasKey(jbbStr) staticVal = gDefines.hasKey(staticStr) verVal = if gDefines.hasKey(verStr): @@ -1137,16 +1228,19 @@ macro getHeader*( `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 + `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) elif `nameConan`: getConanPath(header, conanuri, outdir, version, shared) + elif `nameJBB`: + getJBBPath(header, jbburi, outdir, version) else: getLocalPath(header, outdir) @@ -1171,10 +1265,10 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - # Run preBuild hook before building library if not Std or Conan - when not (useStd or `nameConan`) and declared(`preBuild`): + # Run preBuild hook before building library if not Std, Conan or JBB + when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): static: `preBuild`(`outdir`, prePath) @@ -1183,28 +1277,36 @@ macro getHeader*( `lpath`* = when useStd: stdLPath - elif `nameConan`: - when `nameStatic`: - getConanLPath(`outdir`) - else: - findFile(`lname`, `outdir`, regex = true) + elif `nameConan` or `nameJBB`: + findFile(`lname`, `outdir`, regex = true) else: buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) + # Library dependecy paths + `ldeps`*: seq[string] = + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] + # Header path - search again in case header is generated in build `path`* = if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & - "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name` + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` - # Automatically link with static libary + # Automatically link with static library and dependencies when `nameStatic`: {.passL: `lpath`.} + if `ldeps`.len != 0: + {.passL: `ldeps`.join(" ").} ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 95ab24f..bfe2e94 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -11,8 +11,8 @@ type bhash*: string shared*: bool - headers*: seq[string] - libs*: seq[string] + sharedLibs*: seq[string] + staticLibs*: seq[string] requires*: seq[ConanPackage] ConanBuild* = ref object @@ -26,14 +26,14 @@ type const # Conan API urls - baseUrl = "https://conan.bintray.com/v2/conans" - searchUrl = baseUrl & "/search?q=$query" - pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query" - cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" - dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" + conanBaseUrl = "https://conan.bintray.com/v2/conans" + conanSearchUrl = conanBaseUrl & "/search?q=$query" + conanPkgUrl = conanBaseUrl & "/$name/$version/$user/$channel/search$query" + conanCfgUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" + conanDlUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" # Bintray download sub-URL for explicit `user/channel` (not _/_) - dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" + conanDlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" # Strings conanInfo = "conaninfo.json" @@ -42,18 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - baseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compiletime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - -proc addAltBaseUrl*(name, url: string) = +proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray - baseAltUrl[name] = url + conanBaseAltUrl[name] = url proc jsonGet(url: string): JsonNode = # Make HTTP call and return content as JSON @@ -137,7 +133,7 @@ proc getUriFromConanPackage*(pkg: ConanPackage): string = proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = ## Search for package by `name` and optional `version`, `user` and `channel` - ## + ## ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name @@ -149,7 +145,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac query &= "/" & channel let - j1 = jsonGet(searchUrl % ["query", query]) + j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() if res.len != 0: @@ -193,7 +189,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query.replace("&", "%20and%20") else: "" - url = pkgUrl % [ + url = conanPkgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -234,7 +230,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = ## Get all revisions of a build let - url = cfgUrl % [ + url = conanCfgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -269,22 +265,23 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = writeFile(file, $$pkg) proc parseConanManifest(pkg: ConanPackage, outdir: string) = - # Get all header and library info from downloaded conan package + # Get all library info from downloaded conan package let file = outdir / conanManifest - + if fileExists(file): let data = readFile(file) for line in data.splitLines(): let line = line.split(':')[0] - if line.startsWith("include/"): - pkg.headers.add line - elif line.startsWith("lib/"): - pkg.libs.add line + if line.startsWith("lib/"): + if line.endsWith(".a") or line.endsWith(".lib"): + pkg.staticLibs.add line + elif line.endsWith(".so"): + pkg.sharedLibs.add line elif line.startsWith("bin/") and line.endsWith("dll"): - pkg.libs.add line + pkg.sharedLibs.add line proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") = ## Download specific `revision` of `bld` to `outdir` @@ -300,7 +297,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision url = if pkg.user == "_": - dlUrl % [ + conanDlUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -311,7 +308,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision "file", conanPackage ] else: - baseAltUrl[pkg.user] & dlAltUrl % [ + conanBaseAltUrl[pkg.user] & conanDlAltUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -331,7 +328,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` - ## + ## ## High-level API that handles the end to end Conan process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() @@ -351,7 +348,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = pkg.getConanBuilds() - doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" echo &"# Downloading {pkg.name} v{pkg.version} from Conan" for recipe, builds in pkg.recipes: @@ -375,18 +372,25 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req) + rpkg = newConanPackageFromUri(req, shared = false) downloadConan(rpkg, outdir, clean = false) pkg.requires.add rpkg -proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = - ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps +proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = + ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps ## in descending order - ## + ## ## `outdir` is prefixed to each entry - for lib in pkg.libs: - result.add outdir / lib + let + libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs + str = if pkg.shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib for cpkg in pkg.requires: - result.add cpkg.getConanLibs(outdir) + result.add cpkg.getConanLDeps(outdir, main = false) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim new file mode 100644 index 0000000..9f0d173 --- /dev/null +++ b/nimterop/jbb.nim @@ -0,0 +1,210 @@ +import marshal, os, strutils + +type + JBBPackage* = ref object + ## JBBPackage type that stores package information + name*: string + version*: string + + url*: string + + sharedLibs*: seq[string] + staticLibs*: seq[string] + requires*: seq[JBBPackage] + +const + # JBB URLs + jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl" + + jbbInfo = "jbbinfo.json" + jbbProject = "Project.toml" + jbbArtifacts = "Artifacts.toml" + +proc `==`*(pkg1, pkg2: JBBPackage): bool = + ## Check if two JBBPackage objects are equal + (not pkg1.isNil and not pkg2.isNil and + pkg1.name == pkg2.name and + pkg1.version == pkg2.version) + +proc newJBBPackage*(name, version: string): JBBPackage = + ## Create a new JBBPackage with specified name and version + result = new(JBBPackage) + result.name = name + result.version = version + +proc parseJBBProject(pkg: JBBPackage, outdir: string) = + # Get all dependencies from Project.toml + let + file = outdir / jbbProject + + if fileExists(file): + let + data = readFile(file) + var + deps = false + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith('['): + if line == "[deps]": + deps = true + else: + deps = false + elif deps: + let + name = line.split()[0] + if name.endsWith("_jll"): + pkg.requires.add newJBBPackage(name[0 .. ^5], "") + +proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = + # Get build information from Artifacts.toml + let + file = outdir / jbbArtifacts + + (arch, os, _, _) = getGccInfo() + + if fileExists(file): + let + data = readFile(file) + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + var + found = false + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith("arch = ") and not found: + let + barch = line.split(" = ")[1].strip(chars = {'"'}) + if barch == arch: + found = true + elif line.startsWith("os = ") and found: + let + bos = line.split(" = ")[1].strip(chars = {'"'}) + if bos != os: + found = false + elif line.startsWith("url = ") and found: + pkg.url = line.split(" = ")[1].strip(chars = {'"'}) + break + +proc findJBBLibs(pkg: JBBPackage, outdir: string) = + pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir) + pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir) + for i in 0 ..< pkg.sharedLibs.len: + if pkg.sharedLibs[i].isAbsolute: + pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1] + + for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): + if not lib.endsWith(".dll.a"): + if lib.isAbsolute: + pkg.staticLibs.add lib[outdir.len+1 .. ^1] + else: + pkg.staticLibs.add lib + +proc getJBBRepo*(pkg: JBBPackage, outdir: string) = + ## Clone JBB package repo and checkout version tag if version is + ## specified in package + let + path = outdir / "repos" / pkg.name + + gitPull( + jbbBaseUrl % pkg.name, + outdir = path, + plist = "*.toml", + "master", + quiet = true + ) + + if pkg.version.len != 0: + # Checkout correct tag + let + tags = gitTags(path) + for i in tags.len - 1 .. 0: + if pkg.version in tags[i] and i != tags.len - 1: + gitCheckout(path, tags[i-1]) + + pkg.parseJBBProject(path) + pkg.parseJBBArtifacts(path) + +proc loadJBBInfo*(outdir: string): JBBPackage = + ## Load cached package info from `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + if fileExists(file): + result = to[JBBPackage](readFile(file)) + +proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = + ## Save downloaded package info to `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + writeFile(file, $$pkg) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) +proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = + ## Download `pkg` from BinaryBuilder.org to `outdir` + ## + ## High-level API that handles the end to end JBB process flow to find + ## latest package binary and downloads and extracts it to `outdir`. + fixOutDir() + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return + elif clean: + cleanDir(outdir) + + pkg.getJBBRepo(outdir) + + doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org" + + let + vstr = + if pkg.version.len != 0: + &" v{pkg.version}" + else: + "" + path = outdir / "downloads" / pkg.name + echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" + downloadUrl(pkg.url, path, quiet = true) + pkg.findJBBLibs(path) + mvTree(path, outdir) + + pkg.dlJBBRequires(outdir) + + if clean: + pkg.saveJBBInfo(outdir) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = + ## Download all required dependencies of this `pkg` + fixOutDir() + for rpkg in pkg.requires: + downloadJBB(rpkg, outdir, clean = false) + +proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = + ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps + ## in descending order + ## + ## `outdir` is prefixed to each entry + let + libs = if shared: pkg.sharedLibs else: pkg.staticLibs + str = if shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib + + for cpkg in pkg.requires: + result.add cpkg.getJBBLDeps(outdir, shared, main = false) diff --git a/tests/getheader.nims b/tests/getheader.nims index 349b9bb..1d0ba1d 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -23,6 +23,7 @@ var testCall(cmd & lrcmd, "No build files found", 1) testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) +testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) when defined(posix): # stdlib @@ -43,6 +44,11 @@ else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) +# JBB +testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) diff --git a/tests/libssh2.nim b/tests/libssh2.nim index b7e591c..48b8ba9 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,6 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", + jbburi = "libssh2/$1", outdir = outdir ) @@ -14,16 +15,16 @@ cOverride: stat = object stat64 = object SOCKET = object - + when not libssh2Static: cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} {.passL: "-lpthread".} @@ -39,8 +40,11 @@ if session == nil: libssh2_session_set_blocking(session, 0.cint) echo "zlib version = " & (block: - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): $zlibVersion() else: "" -) \ No newline at end of file +) + +static: + echo libssh2LDeps \ No newline at end of file diff --git a/tests/zlib.nim b/tests/zlib.nim index 15400d4..2bd9395 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -28,6 +28,7 @@ getHeader( giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", conanuri = "zlib/$1", + jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) @@ -72,3 +73,6 @@ else: cImport(zlibPath, recurse = true, flags = FLAGS) echo "zlib version = " & $zlibVersion() + +when isDefined(zlibJBB) and isDefined(zlibStatic): + {.passL: "-no-pie".} From 9d8fee19c01881619b68164a51aaf51dd17e8be3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 17:41:18 -0500 Subject: [PATCH 177/255] libdir support --- nimterop/build.nim | 97 +++++++++++++++++++++++++++++++++++++++----- nimterop/cimport.nim | 13 ------ nimterop/conan.nim | 2 +- nimterop/jbb.nim | 16 ++------ nimterop/nimconf.nim | 61 +++++++++++++--------------- 5 files changed, 120 insertions(+), 69 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index eda27e7..6cb497d 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -158,8 +158,10 @@ proc mkDir*(dir: string) = flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) -proc cpFile*(source, dest: string, move = false) = +proc cpFile*(source, dest: string, psymlink = false, move = false) = ## Copy a file from `source` to `dest` at compile time + ## + ## `psymlink = true` preserves symlinks instead of dereferencing on posix let source = source.replace("/", $DirSep) dest = dest.replace("/", $DirSep) @@ -173,7 +175,10 @@ proc cpFile*(source, dest: string, move = false) = if move: "mv -f" else: - "cp -f" + if psymlink: + "cp -fd" + else: + "cp -f" discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) @@ -233,6 +238,20 @@ proc mvTree*(source, dest: string) = ## Move contents of source dir to the destination, not the directory itself cpTree(source, dest, move = true) +proc getFileDate*(fullpath: string): string = + ## Get file date for `fullpath` + var + ret = 0 + cmd = + when defined(Windows): + &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" + elif defined(Linux): + &"stat -c %y {fullpath.sanitizePath}" + elif defined(OSX) or defined(FreeBSD): + &"stat -f %m {fullpath.sanitizePath}" + + (result, ret) = execAction(cmd) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -1083,7 +1102,8 @@ macro isDefined*(def: untyped): untyped = macro getHeader*( header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", + outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -1125,7 +1145,14 @@ macro getHeader*( ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. + ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. + ## + ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their + ## dependencies to that directory. This prevents any runtime failures if `outdir` gets + ## removed or its contents changed. By default, `libdir` is set to the output directory + ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will + ## reflect this new location. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note @@ -1214,6 +1241,8 @@ macro getHeader*( "" mode = getCompilerMode(header) + libdir = if libdir.len != 0: libdir else: getOutDir() + # Use alternate library names if specified for regex search if altNames.len != 0: lre = lre % ("(" & altNames.replace(",", "|") & ")") @@ -1272,9 +1301,9 @@ macro getHeader*( static: `preBuild`(`outdir`, prePath) - const - # Library binary path - build if not standard - `lpath`* = + let + # Library binary path - build if not standard / conan / jbb + lpath {.compiletime.} = when useStd: stdLPath elif `nameConan` or `nameJBB`: @@ -1283,7 +1312,7 @@ macro getHeader*( buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Library dependecy paths - `ldeps`*: seq[string] = + ldeps {.compiletime.}: seq[string] = when `nameConan`: getConanLDeps(`outdir`) elif `nameJBB`: @@ -1291,6 +1320,7 @@ macro getHeader*( else: @[] + const # Header path - search again in case header is generated in build `path`* = if prePath.len != 0: @@ -1301,12 +1331,57 @@ macro getHeader*( static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` - doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" - echo "# Including library " & `lpath` + doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" + + proc extractFilenameStatic(str: string): string {.compiletime.} = + var + pos = -1 + for i in countdown(str.len - 1, 0): + if str[i] == '/' or str[i] == '\\': + pos = i + 1 + break + result = str[pos .. ^1] + + proc joinPathStatic(str1, str2: string): string {.compiletime.} = + let + sep = when defined(Windows): "\\" else: "/" + result = str1 & sep & str2 - # Automatically link with static library and dependencies when `nameStatic`: + const + `lpath`* = lpath + `ldeps`* = ldeps + + # Automatically link with static library and dependencies {.passL: `lpath`.} if `ldeps`.len != 0: {.passL: `ldeps`.join(" ").} + + static: + echo "# Including library " & lpath + if `ldeps`.len != 0: + echo "# Including dependencies " & `ldeps`.join(" ") + else: + const + `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic()) + `ldeps`* = block: + var + ldeps = ldeps + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilenameStatic() + ldeptgt = joinPathStatic(`libdir`, lname) + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) > getFileDate(ldeptgt): + echo "# Copying " & lname & " to " & `libdir` + cpFile(ldeps[i], ldeptgt, psymlink = true) + ldeps[i] = ldeptgt + ldeps + + static: + # Copy shared libraries and dependencies to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) > getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` + cpFile(lpath, `lpath`) + + echo "# Including library " & `lpath` ) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 0559db7..c81c35b 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -70,19 +70,6 @@ proc walkDirImpl(indir, inext: string, file=true): seq[string] = if ret == 0: result = output.splitLines() -proc getFileDate(fullpath: string): string = - var - ret = 0 - cmd = - when defined(Windows): - &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" - elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" - elif defined(OSX) or defined(FreeBSD): - &"stat -f %m {fullpath.sanitizePath}" - - (result, ret) = execAction(cmd) - proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index bfe2e94..f8740d0 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -278,7 +278,7 @@ proc parseConanManifest(pkg: ConanPackage, outdir: string) = if line.startsWith("lib/"): if line.endsWith(".a") or line.endsWith(".lib"): pkg.staticLibs.add line - elif line.endsWith(".so"): + elif line.endsWith(".so") or line.endsWith(".dylib"): pkg.sharedLibs.add line elif line.startsWith("bin/") and line.endsWith("dll"): pkg.sharedLibs.add line diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 9f0d173..3e7e623 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -94,18 +94,11 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = break proc findJBBLibs(pkg: JBBPackage, outdir: string) = - pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir) - pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir) - for i in 0 ..< pkg.sharedLibs.len: - if pkg.sharedLibs[i].isAbsolute: - pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1] + pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dynlib)[0-9.]*", outdir) for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): if not lib.endsWith(".dll.a"): - if lib.isAbsolute: - pkg.staticLibs.add lib[outdir.len+1 .. ^1] - else: - pkg.staticLibs.add lib + pkg.staticLibs.add lib proc getJBBRepo*(pkg: JBBPackage, outdir: string) = ## Clone JBB package repo and checkout version tag if version is @@ -174,11 +167,10 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = &" v{pkg.version}" else: "" - path = outdir / "downloads" / pkg.name + path = outdir / pkg.name echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" downloadUrl(pkg.url, path, quiet = true) pkg.findJBBLibs(path) - mvTree(path, outdir) pkg.dlJBBRequires(outdir) @@ -204,7 +196,7 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s if not main: for lib in libs: - result.add outdir / lib + result.add lib for cpkg in pkg.requires: result.add cpkg.getJBBLDeps(outdir, shared, main = false) diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 33215ef..98fc8fe 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -16,6 +16,7 @@ type paths*: OrderedSet[string] nimblePaths*: OrderedSet[string] nimcacheDir*: string + outDir*: string proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` @@ -67,35 +68,6 @@ proc stripName(path, projectName: string): string = else: result = path -proc getNimcacheDir*(projectDir = ""): string = - ## Get nimcache directory for current compilation or specified `projectDir` - when nimvm: - when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): - # Get value at compile time from `std/compilesettings` - result = stripName( - querySetting(SingleValueSetting.nimcacheDir), - querySetting(SingleValueSetting.projectName) - ) - else: - discard - - # Not Nim v1.2.0+ or runtime - if result.len == 0: - let - # Get project directory for < v1.2.0 at compile time - projectDir = if projectDir.len != 0: projectDir else: getProjectDir() - - # Use `nim dump` to figure out nimcache for `projectDir` - let - dumpJson = getJson(projectDir) - - if dumpJson != nil and dumpJson.hasKey("nimcache"): - result = stripName(dumpJson["nimcache"].getStr(), "dummy") - - # Set to OS defaults if not detectable - if result.len == 0: - result = getOsCacheDir() - proc jsonToSeq(node: JsonNode, key: string): seq[string] = # Convert JsonArray to seq[string] for specified `key` if node.hasKey(key): @@ -126,7 +98,11 @@ proc getNimConfig*(projectDir = ""): Config = libPath = getCurrentCompilerExe().parentDir().parentDir() / "lib" lazyPaths = querySettingSeq(MultipleValueSetting.lazyPaths) searchPaths = querySettingSeq(MultipleValueSetting.searchPaths) - result.nimcacheDir = querySetting(SingleValueSetting.nimcacheDir) + result.nimcacheDir = stripName( + querySetting(SingleValueSetting.nimcacheDir), + querySetting(SingleValueSetting.projectName) + ) + result.outDir = querySetting(SingleValueSetting.outDir) else: discard @@ -150,6 +126,11 @@ proc getNimConfig*(projectDir = ""): Config = # Usually `libPath` is last entry in `searchPaths` libPath = searchPaths[^1] + if dumpJson.hasKey("nimcache"): + result.nimcacheDir = stripName(dumpJson["nimcache"].getStr(), "dummy") + if dumpJson.hasKey("outdir"): + result.outDir = dumpJson["outdir"].getStr() + # Parse version if version.len != 0: let @@ -188,7 +169,11 @@ proc getNimConfig*(projectDir = ""): Config = if not skip: result.paths.incl path - result.nimcacheDir = getNimcacheDir(projectDir) + if result.nimcacheDir.len == 0: + result.nimcacheDir = getOsCacheDir() + + if result.outDir.len == 0: + result.outDir = projectDir proc getNimConfigFlags(cfg: Config): string = # Convert configuration into Nim flags for cfg file or command line @@ -227,4 +212,16 @@ proc writeNimConfig*(cfgFile: string, projectDir = "") = let cfg = getNimConfig(projectDir) cfgOut = getNimConfigFlags(cfg) - writeFile(cfgFile, cfgOut) \ No newline at end of file + writeFile(cfgFile, cfgOut) + +proc getNimcacheDir*(projectDir = ""): string = + ## Get nimcache directory for current compilation or specified `projectDir` + let + cfg = getNimConfig(projectDir) + result = cfg.nimcacheDir + +proc getOutDir*(projectDir = ""): string = + ## Get output directory for current compilation or specified `projectDir` + let + cfg = getNimConfig(projectDir) + result = cfg.outDir From c359cc0226effd3b3dc47af74defb6b6b7bc7bca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 20:41:32 -0500 Subject: [PATCH 178/255] Fix lib copy issue --- nimterop/build.nim | 20 +++++++++++++++++--- nimterop/conan.nim | 2 ++ tests/getheader.nims | 3 +++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 6cb497d..b5cdc59 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -252,6 +252,17 @@ proc getFileDate*(fullpath: string): string = (result, ret) = execAction(cmd) +proc touchFile*(fullpath: string) = + ## Touch file to update modified date + var + cmd = + when defined(Windows): + &"cmd /c copy /b {fullpath.sanitizePath}+" + else: + &"touch {fullpath.sanitizePath}" + + discard execAction(cmd) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -1367,19 +1378,22 @@ macro getHeader*( `ldeps`* = block: var ldeps = ldeps + copied: seq[string] for i in 0 ..< ldeps.len: let lname = ldeps[i].extractFilenameStatic() ldeptgt = joinPathStatic(`libdir`, lname) - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) > getFileDate(ldeptgt): - echo "# Copying " & lname & " to " & `libdir` + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname ldeps[i] = ldeptgt + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` ldeps static: # Copy shared libraries and dependencies to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) > getFileDate(`lpath`): + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` cpFile(lpath, `lpath`) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index f8740d0..fcb0495 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -346,6 +346,8 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + pkg.getConanBuilds() doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" diff --git a/tests/getheader.nims b/tests/getheader.nims index 1d0ba1d..7a3c176 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -40,6 +40,7 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) +<<<<<<< HEAD else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) @@ -48,6 +49,8 @@ else: testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) +======= +>>>>>>> c35eb74... Add tests for conan, recurse implies preprocess # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) From 0a014e084f3afef204b7539fa688183d9b2f62d7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 22:57:28 -0500 Subject: [PATCH 179/255] Fix OSX --- nimterop/build.nim | 4 ++-- nimterop/jbb.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index b5cdc59..28f2b3c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -176,7 +176,7 @@ proc cpFile*(source, dest: string, psymlink = false, move = false) = "mv -f" else: if psymlink: - "cp -fd" + "cp -fa" else: "cp -f" @@ -1055,7 +1055,7 @@ proc getDynlibExt(): string = elif defined(linux) or defined(FreeBSD): result = "\\.so[0-9.]*" elif defined(macosx): - result = "\\.dylib[0-9.]*" + result = "[0-9.\\-]*\\.dylib" var gDefines {.compileTime.} = initTable[string, string]() diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 3e7e623..de59a87 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -94,7 +94,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = break proc findJBBLibs(pkg: JBBPackage, outdir: string) = - pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dynlib)[0-9.]*", outdir) + pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): if not lib.endsWith(".dll.a"): From 7870840d2d6992f53d1bfc319e2b52422b234f36 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 23:37:03 -0500 Subject: [PATCH 180/255] Fix Windows file date --- nimterop/build.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 28f2b3c..dbdcd4c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -244,7 +244,9 @@ proc getFileDate*(fullpath: string): string = ret = 0 cmd = when defined(Windows): - &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" + let + (head, tail) = fullpath.splitPath() + &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" elif defined(Linux): &"stat -c %y {fullpath.sanitizePath}" elif defined(OSX) or defined(FreeBSD): From 1820fdffcf8af4b1c5ecca8bdc548c206e24b439 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 10:22:22 -0500 Subject: [PATCH 181/255] Optional version and uri, no Std copy, reuse deps, jbb libc check --- .travis.yml | 2 +- appveyor.yml | 1 + nimterop/build.nim | 150 +++++++++++++++++++++++++++---------------- nimterop/conan.nim | 58 +++++++++++++---- nimterop/globals.nim | 2 +- nimterop/jbb.nim | 63 +++++++++++------- tests/libssh2.nim | 5 +- tests/lzma.nim | 2 + tests/zlib.nim | 2 - 9 files changed, 185 insertions(+), 100 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4e9bde..173da68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 - - BRANCH=1.2.0 + - BRANCH=1.2.2 - BRANCH=devel cache: diff --git a/appveyor.yml b/appveyor.yml index d67b01d..b572ea1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,7 @@ environment: matrix: - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.6 + - NIM_VERSION: 1.2.2 for: - diff --git a/nimterop/build.nim b/nimterop/build.nim index dbdcd4c..b391f7a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep +export extractFilename, `/` + type BuildType* = enum btAutoconf, btCmake @@ -850,6 +852,31 @@ template fixOutDir() {.dirty.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir +proc compareVersions*(ver1, ver2: string): int = + ## Compare two version strings x.y.z and return -1, 0, 1 + ## + ## ver1 < ver2 = -1 + ## ver1 = ver2 = 0 + ## ver1 > ver2 = 1 + let + ver1seq = ver1.replace("-", "").split('.') + ver2seq = ver2.replace("-", "").split('.') + for i in 0 ..< ver1seq.len: + let + p1 = ver1seq[i] + p2 = if i < ver2seq.len: ver2seq[i] else: "0" + + try: + let + h1 = p1.parseHexInt() + h2 = p2.parseHexInt() + + if h1 < h2: return -1 + elif h1 > h2: return 1 + except ValueError: + if p1 < p2: return -1 + elif p1 > p2: return 1 + # Conan support include conan @@ -913,10 +940,10 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = uri = uri if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan uri" + doAssert version.len != 0, "Need version for Conan.io uri: " & uri uri = uri % version - else: - doAssert version.len == 0, "Conan uri does not contain version" + elif version.len != 0: + uri = uri & "/" & version let pkg = newConanPackageFromUri(uri, shared) @@ -934,6 +961,7 @@ proc getJBBPath(header, uri, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] + hasVersion = version.len != 0 var ver = @@ -942,11 +970,15 @@ proc getJBBPath(header, uri, outdir, version: string): string = else: "" - if "$#" in ver or "$1" in ver: - doAssert version.len != 0, "Need version for BinaryBuilder.org uri" - ver = ver % version - else: - doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + if ver.len != 0: + if "$#" in ver or "$1" in ver: + doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri + ver = ver % version + elif hasVersion: + doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & + version & "\"` for BinaryBuilder.org" + elif hasVersion: + ver = version let pkg = newJBBPackage(name, ver) @@ -1129,10 +1161,10 @@ macro getHeader*( ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux ## `-d:xxxGit` - clone source from a git repo specified in `giturl` ## `-d:xxxDL` - download source from `dlurl` and extract if required - ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` - ## typically formatted as `name/version[@user/channel][:bhash]` - ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` - ## typically formatted as `name/version` + ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with + ## format `pkgname[/version[@user/channel][:bhash]]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with + ## format `pkgname[/version]` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. @@ -1145,8 +1177,15 @@ macro getHeader*( ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL - ## defined. + ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL + ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for + ## `Conan` and `JBB` if available. If no version is specified, the latest release of the + ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional + ## URI information: + ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` + ## + ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` + ## filename is used instead. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1165,7 +1204,7 @@ macro getHeader*( ## dependencies to that directory. This prevents any runtime failures if `outdir` gets ## removed or its contents changed. By default, `libdir` is set to the output directory ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will - ## reflect this new location. + ## reflect this new location. `libdir` is ignored for `Std` mode. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note @@ -1210,6 +1249,10 @@ macro getHeader*( origname = header.extractFilename().split(".")[0] name = origname.split(seps = AllChars-Letters-Digits).join() + # Default to origname if not specified + conanuri = if conanuri.len != 0: conanuri else: origname + jbburi = if jbburi.len != 0: jbburi else: origname + # -d:xxx for this header stdStr = name & "Std" gitStr = name & "Git" @@ -1316,7 +1359,7 @@ macro getHeader*( let # Library binary path - build if not standard / conan / jbb - lpath {.compiletime.} = + lpath {.compileTime.} = when useStd: stdLPath elif `nameConan` or `nameJBB`: @@ -1325,11 +1368,14 @@ macro getHeader*( buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Library dependecy paths - ldeps {.compiletime.}: seq[string] = - when `nameConan`: - getConanLDeps(`outdir`) - elif `nameJBB`: - getJBBLDeps(`outdir`, not `nameStatic`) + ldeps {.compileTime.}: seq[string] = + when not useStd: + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] else: @[] @@ -1346,20 +1392,6 @@ macro getHeader*( "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" - proc extractFilenameStatic(str: string): string {.compiletime.} = - var - pos = -1 - for i in countdown(str.len - 1, 0): - if str[i] == '/' or str[i] == '\\': - pos = i + 1 - break - result = str[pos .. ^1] - - proc joinPathStatic(str1, str2: string): string {.compiletime.} = - let - sep = when defined(Windows): "\\" else: "/" - result = str1 & sep & str2 - when `nameStatic`: const `lpath`* = lpath @@ -1376,28 +1408,34 @@ macro getHeader*( echo "# Including dependencies " & `ldeps`.join(" ") else: const - `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic()) - `ldeps`* = block: - var - ldeps = ldeps - copied: seq[string] - for i in 0 ..< ldeps.len: - let - lname = ldeps[i].extractFilenameStatic() - ldeptgt = joinPathStatic(`libdir`, lname) - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): - cpFile(ldeps[i], ldeptgt, psymlink = true) - copied.add lname - ldeps[i] = ldeptgt - if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` - ldeps + `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath + `ldeps`* = + when not useStd: + block: + var + ldeps = ldeps + copied: seq[string] + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilename() + ldeptgt = `libdir` / lname + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): + cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname + ldeps[i] = ldeptgt + # Copy downloaded dependencies to `libdir` + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + ldeps + else: + ldeps static: - # Copy shared libraries and dependencies to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` - cpFile(lpath, `lpath`) + when not useStd: + # Copy downloaded shared libraries to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + cpFile(lpath, `lpath`) - echo "# Including library " & `lpath` + echo "# Including library " & `lpath` ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index fcb0495..6e761df 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -42,11 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - conanBaseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compileTime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() + # Reuse dependencies already downloaded + gConanRequires {.compileTime.}: Table[string, ConanPackage] + proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray conanBaseAltUrl[name] = url @@ -144,13 +147,28 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if channel.len != 0: query &= "/" & channel + echo &"# Searching Conan.io for latest version of {name}" + let j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() - if res.len != 0: - # Return last entry - latest - result = newConanPackageFromUri(res[^1].getStr()) + # Return latest comparing versions - prefer @_/_ + var + latest = "" + latestv = "" + for i in 0 ..< res.len: + let + str = res[i].getStr() + if "@_/_" in str: + let + ver = str.split('/')[1].split('@')[0] + if latestv.len == 0 or compareVersions(ver, latestv) > 0: + latestv = ver + latest = str + + if latest.len != 0: + result = newConanPackageFromUri(latest) proc searchConan*(pkg: ConanPackage): ConanPackage = ## Search for latest package based on incomplete package info @@ -288,6 +306,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## ## If omitted, the latest revision (first) is downloaded fixOutDir() + + doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() + let revision = if revision.len != 0: @@ -326,7 +347,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision rmFile(outdir / conanManifest) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) -proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = +proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` ## ## High-level API that handles the end to end Conan process flow to find @@ -339,11 +360,13 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = else: pkg - cpkg = loadConanInfo(outdir) + if main: + let + cpkg = loadConanInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) echo &"# Downloading {pkg.name} v{pkg.version} from Conan" @@ -352,7 +375,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: @@ -362,7 +385,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = break break - if clean: + if main: pkg.saveConanInfo(outdir) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = @@ -374,10 +397,17 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req, shared = false) + name = req.split('/')[0] + if gConanRequires.hasKey(name): + # Reuse dep already downloaded + pkg.requires.add gConanRequires[name] + else: + let + rpkg = newConanPackageFromUri(req, shared = false) - downloadConan(rpkg, outdir, clean = false) - pkg.requires.add rpkg + downloadConan(rpkg, outdir, main = false) + pkg.requires.add rpkg + gConanRequires[name] = rpkg proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 8062ac8..c79f4a5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -146,7 +146,7 @@ when defined(TOAST): gecho join(args, "").getCommented() else: var - gStateCT* {.compiletime, used.} = new(State) + gStateCT* {.compileTime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index de59a87..5d421f0 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -20,6 +20,10 @@ const jbbProject = "Project.toml" jbbArtifacts = "Artifacts.toml" +var + # Reuse dependencies already downloaded + gJBBRequires {.compileTime.}: Table[string, JBBPackage] + proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -79,19 +83,26 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = let line = line.strip() if line.len != 0: - if line.startsWith("arch = ") and not found: - let - barch = line.split(" = ")[1].strip(chars = {'"'}) - if barch == arch: - found = true - elif line.startsWith("os = ") and found: - let - bos = line.split(" = ")[1].strip(chars = {'"'}) - if bos != os: - found = false - elif line.startsWith("url = ") and found: - pkg.url = line.split(" = ")[1].strip(chars = {'"'}) - break + let + spl = line.split(" = ", 1) + name = spl[0] + val = if spl.len == 2: spl[1].strip(chars = {'"', ' '}) else: "" + + # Match arch, os and glibc on Linux to find download URL + case name + of "arch": + if val == arch and not found: found = true + of "os": + if val != os and found: found = false + of "libc": + when defined(Linux): + if val != "glibc" and found: found = false + of "url": + if found: + pkg.url = val + break + else: + discard proc findJBBLibs(pkg: JBBPackage, outdir: string) = pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) @@ -143,18 +154,19 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = writeFile(file, $$pkg) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) -proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = +proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = ## Download `pkg` from BinaryBuilder.org to `outdir` ## ## High-level API that handles the end to end JBB process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() - let - cpkg = loadJBBInfo(outdir) + if main: + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) pkg.getJBBRepo(outdir) @@ -174,14 +186,21 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = pkg.dlJBBRequires(outdir) - if clean: + if main: pkg.saveJBBInfo(outdir) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = ## Download all required dependencies of this `pkg` fixOutDir() - for rpkg in pkg.requires: - downloadJBB(rpkg, outdir, clean = false) + for i in 0 ..< pkg.requires.len: + let + rpkg = pkg.requires[i] + if gJBBRequires.hasKey(rpkg.name): + # Reuse dep already downloaded + pkg.requires[i] = gJBBRequires[rpkg.name] + else: + downloadJBB(rpkg, outdir, main = false) + gJBBRequires[rpkg.name] = rpkg proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 48b8ba9..172f6ae 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,7 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", - jbburi = "libssh2/$1", + jbburi = "libssh2/1.9.0", outdir = outdir ) @@ -45,6 +45,3 @@ echo "zlib version = " & (block: else: "" ) - -static: - echo libssh2LDeps \ No newline at end of file diff --git a/tests/lzma.nim b/tests/lzma.nim index cff39de..9bda18e 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -24,6 +24,8 @@ getHeader( "lzma.h", giturl = "https://github.com/xz-mirror/xz", dlurl = "https://tukaani.org/xz/xz-$1.tar.gz", + conanuri = "xz_utils", + jbburi = "xz", outdir = baseDir, conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo" ) diff --git a/tests/zlib.nim b/tests/zlib.nim index 2bd9395..53384c3 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -27,8 +27,6 @@ getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", - conanuri = "zlib/$1", - jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) From f207911d38537fbadf42069b679259aed2dd929d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 14:33:32 -0500 Subject: [PATCH 182/255] --gc:arc fixes and testing --- nimterop/conan.nim | 9 ++++++--- nimterop/jbb.nim | 9 ++++++--- tests/getheader.nims | 3 +++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 6e761df..ee74141 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -1,4 +1,4 @@ -import json, marshal, os, strformat, strutils, tables +import json, os, strformat, strutils, tables type ConanPackage* = ref object @@ -272,7 +272,10 @@ proc loadConanInfo*(outdir: string): ConanPackage = file = outdir / conanInfo if fileExists(file): - result = to[ConanPackage](readFile(file)) + try: + result = to(readFile(file).parseJson(), ConanPackage) + except: + discard proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` @@ -280,7 +283,7 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = let file = outdir / conanInfo - writeFile(file, $$pkg) + writeFile(file, $(%pkg)) proc parseConanManifest(pkg: ConanPackage, outdir: string) = # Get all library info from downloaded conan package diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 5d421f0..b5e66fb 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -1,4 +1,4 @@ -import marshal, os, strutils +import json, os, strutils type JBBPackage* = ref object @@ -143,7 +143,10 @@ proc loadJBBInfo*(outdir: string): JBBPackage = file = outdir / jbbInfo if fileExists(file): - result = to[JBBPackage](readFile(file)) + try: + result = to(readFile(file).parseJson(), JBBPackage) + except: + discard proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` @@ -151,7 +154,7 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = let file = outdir / jbbInfo - writeFile(file, $$pkg) + writeFile(file, $(%pkg)) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = diff --git a/tests/getheader.nims b/tests/getheader.nims index 7a3c176..2fe4d9c 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -21,6 +21,9 @@ var lexp = "liblzma version = " zexp = "zlib version = " +when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + cmd &= " --gc:arc" + testCall(cmd & lrcmd, "No build files found", 1) testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) From 9b2aa7d4b1691db69e76bad1b681eafc5bfb060a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 14:52:27 -0500 Subject: [PATCH 183/255] Use marshal pre 1.2.0 --- nimterop/conan.nim | 23 +++++++++++++++++------ nimterop/jbb.nim | 19 ++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index ee74141..cc060f8 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -1,4 +1,9 @@ -import json, os, strformat, strutils, tables +import os, strformat, strutils, tables + +when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + import marshal +else: + import json type ConanPackage* = ref object @@ -272,10 +277,13 @@ proc loadConanInfo*(outdir: string): ConanPackage = file = outdir / conanInfo if fileExists(file): - try: - result = to(readFile(file).parseJson(), ConanPackage) - except: - discard + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + result = to[ConanPackage](readFile(file)) + else: + try: + result = to(readFile(file).parseJson(), ConanPackage) + except: + discard proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` @@ -283,7 +291,10 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = let file = outdir / conanInfo - writeFile(file, $(%pkg)) + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + writeFile(file, $$pkg) + else: + writeFile(file, $(%pkg)) proc parseConanManifest(pkg: ConanPackage, outdir: string) = # Get all library info from downloaded conan package diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index b5e66fb..c282197 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -1,5 +1,8 @@ import json, os, strutils +when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + import marshal + type JBBPackage* = ref object ## JBBPackage type that stores package information @@ -143,10 +146,13 @@ proc loadJBBInfo*(outdir: string): JBBPackage = file = outdir / jbbInfo if fileExists(file): - try: - result = to(readFile(file).parseJson(), JBBPackage) - except: - discard + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + result = to[JBBPackage](readFile(file)) + else: + try: + result = to(readFile(file).parseJson(), JBBPackage) + except: + discard proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` @@ -154,7 +160,10 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = let file = outdir / jbbInfo - writeFile(file, $(%pkg)) + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + writeFile(file, $$pkg) + else: + writeFile(file, $(%pkg)) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = From 7e0b0b1a2b69a835a680afce88f7af39b2c944b0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 13:32:01 -0500 Subject: [PATCH 184/255] Retries, conan VS MD/14 --- nimterop/build.nim | 8 ++++---- nimterop/conan.nim | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index b391f7a..edbe439 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -129,7 +129,7 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, # On failure, retry or die as requested if result.ret != 0: if retry > 0: - sleep(500) + sleep(1000) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) elif die: doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & @@ -368,7 +368,7 @@ proc downloadUrl*(url, outdir: string, quiet = false) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) + discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 3) if ext == ".zip": extractZip(file, outdir, quiet) @@ -436,12 +436,12 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false if checkout.len != 0: if not quiet: echo "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 1) + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: if not quiet: echo "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1) + discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) proc gitTags*(outdir: string): seq[string] = ## Get all the git tags in the specified directory diff --git a/nimterop/conan.nim b/nimterop/conan.nim index cc060f8..7e1bd8e 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -208,6 +208,10 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query &= &"&{filter}" if "compiler=" notin filter and os != "windows": query &= &"&compiler={compiler}&compiler.version=" & vfilter + if "compiler.runtime=" notin filter and os == "windows": + query &= &"&compiler.runtime=MD" + if "compiler.version=" notin filter and os == "windows": + query &= &"&compiler.version=14" query.replace("&", "%20and%20") else: "" From 7c9d826998e7347e613d823ba00ca6160d712a2f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 18:03:37 -0500 Subject: [PATCH 185/255] Handle octal values correctly --- nimterop/exprparser.nim | 5 +++-- tests/include/tast2.h | 1 + tests/tast2.nim | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 0fea184..0d2099a 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -140,8 +140,6 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = var val: BiggestInt flags: TNodeFlags - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. if number.startsWith("0X") or number.startsWith("0x"): val = parseHexInt(number) flags = {nfBase16} @@ -203,6 +201,9 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = if number.startsWith("-"): number = number[1 ..< number.len] prefix = "-" + if number.len > 1 and number[0] == '0' and number[1] notin ['x', 'X']: + # Octal 0123 + number = "0o" & number[1 .. ^1] if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): suffix = number[^3 .. ^1] number = number[0 ..< ^3] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 019f7c4..6f569d5 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -7,6 +7,7 @@ extern "C" { #define C 0x10 #define D "hello" #define E 'c' +#define F 01234 #define UEXPR (1234u << 1) #define ULEXPR (1234ul << 2) diff --git a/tests/tast2.nim b/tests/tast2.nim index fd0175a..124c4e4 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -110,6 +110,7 @@ assert B == 1.0 assert C == 0x10 assert D == "hello" assert E == 'c' +assert F == 0o1234 assert not defined(NOTSUPPORTEDSTR) From 73fec6753fdcade0277c6c529e52ef01096e28e1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 21:31:22 -0500 Subject: [PATCH 186/255] Download retries --- nimterop/build.nim | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index edbe439..62b5e1f 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -76,7 +76,7 @@ proc getNimteropCacheDir(): string = result = getNimcacheDir() / "nimterop" proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = ""): tuple[output: string, ret: int] = + cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time ## ## Checks if command exits successfully before returning. If not, an @@ -129,7 +129,9 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, # On failure, retry or die as requested if result.ret != 0: if retry > 0: - sleep(1000) + if not onRetry.isNil: + onRetry() + sleep(500) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) elif die: doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & @@ -344,16 +346,17 @@ proc extractTar*(tarfile, outdir: string, quiet = false) = if name.len != 0: rmFile(outdir / name) -proc downloadUrl*(url, outdir: string, quiet = false) = +proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory ## ## If an archive file, it is automatically extracted after download. let file = url.extractFilename() + filePath = outdir / file ext = file.splitFile().ext.toLowerAscii() archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] - if not (ext in archives and fileExists(outdir/file)): + if not (ext in archives and fileExists(filePath)): if not quiet: echo "# Downloading " & file mkDir(outdir) @@ -368,7 +371,8 @@ proc downloadUrl*(url, outdir: string, quiet = false) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 3) + discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, + onRetry = proc() = rmFile(filePath)) if ext == ".zip": extractZip(file, outdir, quiet) From fbc8c2ad9edd413472849500822d929c6578c568 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 00:09:50 -0500 Subject: [PATCH 187/255] Reorganize file structure --- CHANGES.md | 30 + nimterop/build.nim | 1420 +----------------------- nimterop/build/ccompiler.nim | 101 ++ nimterop/{ => build}/conan.nim | 0 nimterop/build/getheader.nim | 504 +++++++++ nimterop/{ => build}/jbb.nim | 2 +- nimterop/{ => build}/nimconf.nim | 0 nimterop/build/shell.nim | 465 ++++++++ nimterop/build/tools.nim | 260 +++++ nimterop/git.nim | 1 - nimterop/toast.nim | 4 +- nimterop/{ => toastlib}/ast.nim | 3 +- nimterop/{ => toastlib}/ast2.nim | 4 +- nimterop/{ => toastlib}/comphelp.nim | 3 +- nimterop/{ => toastlib}/exprparser.nim | 6 +- nimterop/{ => toastlib}/getters.nim | 2 +- nimterop/{ => toastlib}/grammar.nim | 3 +- nimterop/{ => toastlib}/lisp.nim | 3 +- nimterop/{ => toastlib}/tshelp.nim | 5 +- 19 files changed, 1435 insertions(+), 1381 deletions(-) create mode 100644 nimterop/build/ccompiler.nim rename nimterop/{ => build}/conan.nim (100%) create mode 100644 nimterop/build/getheader.nim rename nimterop/{ => build}/jbb.nim (99%) rename nimterop/{ => build}/nimconf.nim (100%) create mode 100644 nimterop/build/shell.nim create mode 100644 nimterop/build/tools.nim delete mode 100644 nimterop/git.nim rename nimterop/{ => toastlib}/ast.nim (99%) rename nimterop/{ => toastlib}/ast2.nim (99%) rename nimterop/{ => toastlib}/comphelp.nim (98%) rename nimterop/{ => toastlib}/exprparser.nim (99%) rename nimterop/{ => toastlib}/getters.nim (99%) rename nimterop/{ => toastlib}/grammar.nim (99%) rename nimterop/{ => toastlib}/lisp.nim (97%) rename nimterop/{ => toastlib}/tshelp.nim (99%) diff --git a/CHANGES.md b/CHANGES.md index e47974d..cedf5a5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,34 @@ # Nimterop Change History +## Version 0.6.0 + +This release adds the ability to download precompiled binaries from [Conan.io](https://conan.io/center) and Julia's [BinaryBuilder.org](https://binarybuilder.org). This alleviates the headache of searching and downloading libraries manually both for wrapper writers as well as end users. There are some known limitations but it should prove to become more useful as these sites expand their capabilities. + +Conan.io shared builds tend to have all dependencies statically linked into the binary so a single so/dll/dylib has everything. For Conan.io static builds and all libraries on BinaryBuilder.org, dependencies are also downloaded and linked as needed. They are returned in the new `const xxxLDeps` in case wrapper writers need it for some reason. + +Known concerns: +- Conan.io only compiles Windows builds with Microsoft's VC++ compiler so static .lib files may not always work with MinGW on Windows. +- Conan.io compiles all Mac builds on OSX 10.14 so older versions of the OS will grumble when statically linking these libraries. +- BinaryBuilder.org does not include static libs for all their projects. + +Refer to the documentation for `getHeader()` for details on how to use this new capability. + +See the full list of changes here: + +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 + +### Breaking changes + +- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) + +- `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. + +### New functionality + +- `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. + + + ## Version 0.5.0 This release introduces a new backend for wrapper generation dubbed `ast2` that leverages the Nim compiler AST and renderer. The new design simplifies feature development and already includes all the functionality of the legacy algorithm plus fixes for several open issues. @@ -73,6 +102,7 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 [i148]: https://github.com/nimterop/nimterop/issues/148 [i151]: https://github.com/nimterop/nimterop/issues/151 [i153]: https://github.com/nimterop/nimterop/issues/153 +[i154]: https://github.com/nimterop/nimterop/issues/154 [i155]: https://github.com/nimterop/nimterop/issues/155 [i156]: https://github.com/nimterop/nimterop/issues/156 [i159]: https://github.com/nimterop/nimterop/issues/159 diff --git a/nimterop/build.nim b/nimterop/build.nim index 62b5e1f..99adeca 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,21 +4,13 @@ import os except findExe, sleep export extractFilename, `/` -type - BuildType* = enum - btAutoconf, btCmake - - BuildStatus = object - built: bool - buildPath: string - error: string - # build specific debug since we cannot import globals (yet) var gDebug* = false gDebugCT* {.compileTime.} = false gNimExe* = "" +# Misc helpers proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") when nimvm: @@ -26,24 +18,6 @@ proc echoDebug(str: string) = else: if gDebug: echo str -proc fixCmd(cmd: string): string = - when defined(Windows): - # Replace 'cd d:\abc' with 'd: && cd d:\abc` - var filteredCmd = cmd - if cmd.toLower().startsWith("cd"): - var - colonIndex = cmd.find(":") - driveLetter = cmd.substr(colonIndex-1, colonIndex) - if (driveLetter[0].isAlphaAscii() and - driveLetter[1] == ':' and - colonIndex == 4): - filteredCmd = &"{driveLetter} && {cmd}" - result = "cmd /c " & filteredCmd - elif defined(posix): - result = cmd - else: - doAssert false - proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: @@ -57,801 +31,6 @@ proc getCurrentNimCompiler*(): string = else: result = gNimExe -# Nim cfg file related functionality -include "."/nimconf - -proc sleep*(milsecs: int) = - ## Sleep at compile time - let - cmd = - when defined(Windows): - "cmd /c timeout " - else: - "sleep " - - discard gorgeEx(cmd & $(milsecs / 1000)) - -proc getNimteropCacheDir(): string = - # Get location to cache all nimterop artifacts - result = getNimcacheDir() / "nimterop" - -proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = - ## Execute an external command - supported at compile time - ## - ## Checks if command exits successfully before returning. If not, an - ## error is raised. Always caches results to be used in nimsuggest or nimcheck - ## mode. - ## - ## `retry` - number of times command should be retried before error - ## `die = false` - return on errors - ## `cache = true` - cache results unless cleared with -f - ## `cacheKey` - key to create unique cache entry - let - ccmd = fixCmd(cmd) - - when nimvm: - # Cache results for speedup if cache = true - # Else cache for preserving functionality in nimsuggest and nimcheck - let - hash = (ccmd & cacheKey).hash().abs() - cachePath = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash - cacheFile = cachePath & ".txt" - retFile = cachePath & "_ret.txt" - - when defined(nimsuggest) or defined(nimcheck): - # Load results from cache file if generated in previous run - if fileExists(cacheFile) and fileExists(retFile): - result.output = cacheFile.readFile() - result.ret = retFile.readFile().parseInt() - elif die: - doAssert false, "Results not cached - run nim c/cpp at least once\n" & ccmd - else: - if cache and fileExists(cacheFile) and fileExists(retFile) and not compileOption("forceBuild"): - # Return from cache when requested - result.output = cacheFile.readFile() - result.ret = retFile.readFile().parseInt() - else: - # Execute command and store results in cache - (result.output, result.ret) = gorgeEx(ccmd) - if result.ret == 0 or die == false: - # mkdir for execCache dir (circular dependency) - let dir = cacheFile.parentDir() - if not dirExists(dir): - let flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.sanitizePath}") - cacheFile.writeFile(result.output) - retFile.writeFile($result.ret) - else: - # Used by toast - (result.output, result.ret) = execCmdEx(ccmd) - - # On failure, retry or die as requested - if result.ret != 0: - if retry > 0: - if not onRetry.isNil: - onRetry() - sleep(500) - result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) - elif die: - doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & - "\nresult:\n" & result.output - -proc findExe*(exe: string): string = - ## Find the specified executable using the `which`/`where` command - supported - ## at compile time - var - cmd = - when defined(Windows): - "where " & exe - else: - "which " & exe - - (output, ret) = execAction(cmd, die = false) - - if ret == 0: - return output.splitLines()[0].strip() - -proc mkDir*(dir: string) = - ## Create a directory at compile time - ## - ## The `os` module is not available at compile time so a few - ## crucial helper functions are included with nimterop. - if not dirExists(dir): - let - flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) - -proc cpFile*(source, dest: string, psymlink = false, move = false) = - ## Copy a file from `source` to `dest` at compile time - ## - ## `psymlink = true` preserves symlinks instead of dereferencing on posix - let - source = source.replace("/", $DirSep) - dest = dest.replace("/", $DirSep) - cmd = - when defined(Windows): - if move: - "move /y" - else: - "copy /y" - else: - if move: - "mv -f" - else: - if psymlink: - "cp -fa" - else: - "cp -f" - - discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) - -proc mvFile*(source, dest: string) = - ## Move a file from `source` to `dest` at compile time - cpFile(source, dest, move=true) - -proc rmFile*(source: string, dir = false) = - ## Remove a file or pattern at compile time - let - source = source.replace("/", $DirSep) - cmd = - when defined(Windows): - if dir: - "rd /s/q" - else: - "del /s/q/f" - else: - "rm -rf" - exists = - if dir: - dirExists(source) - else: - fileExists(source) - - if exists: - discard execAction(&"{cmd} {source.sanitizePath}", retry = 2) - -proc rmDir*(dir: string) = - ## Remove a directory or pattern at compile time - rmFile(dir, dir = true) - -proc cleanDir*(dir: string) = - ## Remove all contents of a directory at compile time - for kind, path in walkDir(dir): - if kind == pcDir: - rmDir(path) - else: - rmFile(path) - -proc cpTree*(source, dest: string, move = false) = - ## Copy contents of source dir to the destination, not the directory itself - for kind, path in walkDir(source, relative = true): - if kind == pcDir: - cpTree(source / path, dest / path, move) - if move: - rmDir(source / path) - else: - if not dirExists(dest): - mkDir(dest) - if move: - mvFile(source / path, dest / path) - else: - cpFile(source / path, dest / path) - -proc mvTree*(source, dest: string) = - ## Move contents of source dir to the destination, not the directory itself - cpTree(source, dest, move = true) - -proc getFileDate*(fullpath: string): string = - ## Get file date for `fullpath` - var - ret = 0 - cmd = - when defined(Windows): - let - (head, tail) = fullpath.splitPath() - &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" - elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" - elif defined(OSX) or defined(FreeBSD): - &"stat -f %m {fullpath.sanitizePath}" - - (result, ret) = execAction(cmd) - -proc touchFile*(fullpath: string) = - ## Touch file to update modified date - var - cmd = - when defined(Windows): - &"cmd /c copy /b {fullpath.sanitizePath}+" - else: - &"touch {fullpath.sanitizePath}" - - discard execAction(cmd) - -proc getProjectCacheDir*(name: string, forceClean = true): string = - ## Get a cache directory where all nimterop artifacts can be stored - ## - ## Projects can use this location to download source code and build binaries - ## that can be then accessed by multiple apps. This is created under the - ## per-user Nim cache directory. - ## - ## Use `name` to specify the subdirectory name for a project. - ## - ## `forceClean` is enabled by default and effectively deletes the folder - ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows - ## any project to start out with a clean cache dir on a forced build. - ## - ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same - ## `name` when `forceClean = true` else checked out source might get deleted - ## at the wrong time during build. - ## - ## E.g. - ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` - ## - ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so - ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. - result = getNimteropCacheDir() / name - - if forceClean and compileOption("forceBuild"): - echo "# Removing " & result - rmDir(result) - -proc extractZip*(zipfile, outdir: string, quiet = false) = - ## Extract a zip file using `powershell` on Windows and `unzip` on other - ## systems to the specified output directory - var cmd = "unzip -o $#" - if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & - "'System.IO.Compression.FileSystem'; " & - "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - - if not quiet: - echo "# Extracting " & zipfile - discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") - -proc extractTar*(tarfile, outdir: string, quiet = false) = - ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory - var - cmd = "" - name = "" - - if findExe("tar").len != 0: - let - ext = tarfile.splitFile().ext.toLowerAscii() - typ = - case ext - of ".gz", ".tgz": "z" - of ".xz": "J" - of ".bz2": "j" - else: "" - - cmd = "tar xvf" & typ & " " & tarfile.sanitizePath - else: - for i in ["7z", "7za"]: - if findExe(i).len != 0: - cmd = i & " x $#" % tarfile.sanitizePath - - name = tarfile.splitFile().name - if ".tar" in name.toLowerAscii(): - cmd &= " && " & i & " x $#" % name.sanitizePath - - break - - doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath - - if not quiet: - echo "# Extracting " & tarfile - discard execAction(&"cd {outdir.sanitizePath} && {cmd}") - if name.len != 0: - rmFile(outdir / name) - -proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = - ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory - ## - ## If an archive file, it is automatically extracted after download. - let - file = url.extractFilename() - filePath = outdir / file - ext = file.splitFile().ext.toLowerAscii() - archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] - - if not (ext in archives and fileExists(filePath)): - if not quiet: - echo "# Downloading " & file - mkDir(outdir) - var cmd = findExe("curl") - if cmd.len != 0: - cmd &= " -Lk $# -o $#" - else: - cmd = findExe("wget") - if cmd.len != 0: - cmd &= " $# -O $#" - elif defined(Windows): - cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" - else: - doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, - onRetry = proc() = rmFile(filePath)) - - if ext == ".zip": - extractZip(file, outdir, quiet) - elif ext in archives: - extractTar(file, outdir, quiet) - -proc gitReset*(outdir: string) = - ## Hard reset the git repository at the specified directory - echo "# Resetting " & outdir - - let cmd = &"cd {outdir.sanitizePath} && git reset --hard" - while execAction(cmd).output.contains("Permission denied"): - sleep(1000) - echo "# Retrying ..." - -proc gitCheckout*(file, outdir: string) = - ## Checkout the specified `file` in the git repository at `outdir` - ## - ## This effectively resets all changes in the file and can be - ## used to undo any changes that were made to source files to enable - ## successful wrapping with `cImport()` or `c2nImport()`. - echo "# Resetting " & file - let file2 = file.relativePath outdir - let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" - while execAction(cmd).output.contains("Permission denied"): - sleep(500) - echo "# Retrying ..." - -proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = - ## Pull the specified git repository to the output directory - ## - ## `plist` is the list of specific files and directories or wildcards - ## to sparsely checkout. Multiple values can be specified one entry per - ## line. It is optional and if omitted, the entire repository will be - ## checked out. - ## - ## `checkout` is the git tag, branch or commit hash to checkout once - ## the repository is downloaded. This allows for pinning to a specific - ## version of the code. - if dirExists(outdir/".git"): - gitReset(outdir) - return - - let - outdirQ = outdir.sanitizePath - - mkDir(outdir) - - if not quiet: - echo "# Setting up Git repo: " & url - discard execAction(&"cd {outdirQ} && git init .") - discard execAction(&"cd {outdirQ} && git remote add origin {url}") - - if plist.len != 0: - # If a specific list of files is required, create a sparse checkout - # file for git in its config directory - let sparsefile = outdir / ".git/info/sparse-checkout" - - discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") - writeFile(sparsefile, plist) - - # In case directory has old files from another run - discard execAction(&"cd {outdirQ} && git clean -fxd") - - if checkout.len != 0: - if not quiet: - echo "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 3) - discard execAction(&"cd {outdirQ} && git checkout {checkout}") - else: - if not quiet: - echo "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) - -proc gitTags*(outdir: string): seq[string] = - ## Get all the git tags in the specified directory - let - cmd = &"cd {outdir.sanitizePath} && git tag" - tags = execAction(cmd).output.splitLines() - for tag in tags: - let - tag = tag.strip() - if tag.len != 0: - result.add tag - -proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = - ## Find all matching files in the specified directory - ## - ## `file` is a regular expression if `regex` is true - ## - ## Turn off recursive search with `recurse` - var - cmd = - when defined(Windows): - "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" - elif defined(linux): - "find $3 $1 -regextype egrep -regex $2" - elif defined(osx) or defined(FreeBSD): - "find -E $3 $1 -regex $2" - - recursive = "" - - if recurse: - when defined(Windows): - recursive = "--recursive" - else: - when not defined(Windows): - recursive = "-maxdepth 1" - - var - dir = dir - file = file - if not recurse: - let - pdir = file.parentDir() - if pdir.len != 0: - dir = dir / pdir - - file = file.extractFilename - - cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] - - let - (files, ret) = execAction(cmd, die = false) - if ret == 0: - for line in files.splitLines(): - let f = - when defined(Windows): - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" - else: - line - if f.len != 0: - result.add f - -proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = - ## Find the file in the specified directory - ## - ## `file` is a regular expression if `regex` is true - ## - ## Turn off recursive search with `recurse` and stop on first match with - ## `first`. Without it, the shortest match is returned. - let - matches = findFiles(file, dir, recurse, regex) - for match in matches: - if (result.len == 0 or result.len > match.len): - result = match - if first: break - -proc flagBuild*(base: string, flags: openArray[string]): string = - ## Simple helper proc to generate flags for `configure`, `cmake`, etc. - ## - ## Every entry in `flags` is replaced into the `base` string and - ## concatenated to the result. - ## - ## E.g. - ## `base = "--disable-$#"` - ## `flags = @["one", "two"]` - ## - ## `flagBuild(base, flags) => " --disable-one --disable-two"` - for i in flags: - result &= " " & base % i - -proc linkLibs*(names: openArray[string], staticLink = true): string = - ## Create linker flags for specified libraries - ## - ## Prepends `lib` to the name so you only need `ssl` for `libssl`. - var - stat = if staticLink: "--static" else: "" - resSet: OrderedSet[string] - resSet.init() - - for name in names: - let - cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" - (libs, _) = execAction(cmd, die = false) - for lib in libs.split(" "): - resSet.incl lib - - if staticLink: - resSet.incl "--static" - - for res in resSet: - result &= " " & res - -proc configure*(path, check: string, flags = "") = - ## Run the GNU `configure` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## If a `configure` script is not present and an `autogen.sh` script - ## is present, it will be run before attempting `configure`. - ## - ## Next, if `configure.ac` or `configure.in` exist, `autoreconf` will - ## be executed. - ## - ## `check` is a file that will be generated by the `configure` command. - ## This is required to prevent configure from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `configure` command. - if (path / check).fileExists(): - return - - echo "# Configuring " & path - - if not fileExists(path / "configure"): - for i in @["autogen.sh", "build" / "autogen.sh"]: - if fileExists(path / i): - echo "# Running autogen.sh" - - when defined(unix): - echoDebug execAction( - &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output - else: - echoDebug execAction( - &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output - - break - - if not fileExists(path / "configure"): - for i in @["configure.ac", "configure.in"]: - if fileExists(path / i): - echo "# Running autoreconf" - - echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output - - break - - if fileExists(path / "configure"): - echo "# Running configure " & flags - - when defined(unix): - var - cmd = &"cd {path.sanitizePath} && ./configure" - else: - var - cmd = &"cd {path.sanitizePath} && bash ./configure" - if flags.len != 0: - cmd &= &" {flags}" - - echoDebug execAction(cmd).output - - doAssert (path / check).fileExists(), "Configure failed" - -proc getCmakePropertyStr(name, property, value: string): string = - &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" - -proc getCmakeIncludePath*(paths: openArray[string]): string = - ## Create a `cmake` flag to specify custom include paths - ## - ## Result can be included in the `flag` parameter for `cmake()` or - ## the `cmakeFlags` parameter for `getHeader()`. - for path in paths: - result &= path & ";" - result = " -DCMAKE_INCLUDE_PATH=" & result[0 .. ^2].sanitizePath(sep = "/") - -proc setCmakeProperty*(outdir, name, property, value: string) = - ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook - ## for `getHeader()` - ## - ## `set_target_properties(name PROPERTIES property "value")` - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - cm.writeFile( - cm.readFile() & getCmakePropertyStr(name, property, value) - ) - -proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = - ## Set a `cmake` property in `outdir / CMakeLists.txt` to specify a custom library output - ## name - usable in the `xxxPreBuild` hook for `getHeader()` - ## - ## `prefix` is typically `lib` - ## `oname` is the library name - ## `suffix` is typically `.a` - ## - ## Sometimes, `cmake` generates non-standard library names - e.g. zlib compiles to - ## `libzlibstatic.a` on Windows. This proc can help rename it to `libzlib.a` so that `getHeader()` - ## can find it after the library is compiled. - ## - ## ``` - ## set_target_properties(name PROPERTIES PREFIX "prefix") - ## set_target_properties(name PROPERTIES OUTPUT_NAME "oname") - ## set_target_properties(name PROPERTIES SUFFIX "suffix") - ## ``` - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - var - str = "" - if prefix.len != 0: - str &= getCmakePropertyStr(name, "PREFIX", prefix) - if oname.len != 0: - str &= getCmakePropertyStr(name, "OUTPUT_NAME", oname) - if suffix.len != 0: - str &= getCmakePropertyStr(name, "SUFFIX", suffix) - if str.len != 0: - cm.writeFile(cm.readFile() & str) - -proc setCmakePositionIndependentCode*(outdir: string) = - ## Set a `cmake` directive to create libraries with -fPIC enabled - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - let - pic = "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" - cmd = cm.readFile() - if not cmd.contains(pic): - cm.writeFile( - pic & "\n" & cmd - ) - -proc cmake*(path, check, flags: string) = - ## Run the `cmake` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## `path` will be created since typically `cmake` is run in an - ## empty directory. - ## - ## `check` is a file that will be generated by the `cmake` command. - ## This is required to prevent `cmake` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `cmake` command. - ## Unlike `configure`, it is required since typically it will be the - ## path to the repository, typically `..` when `path` is a subdir. - if (path / check).fileExists(): - return - - echo "# Running cmake " & flags - echo "# Path: " & path - - mkDir(path) - - let - cmd = &"cd {path.sanitizePath} && cmake {flags}" - - echoDebug execAction(cmd).output - - doAssert (path / check).fileExists(), "cmake failed" - -proc make*(path, check: string, flags = "", regex = false) = - ## Run the `make` command to build all binaries in the specified path - ## - ## `check` is a file that will be generated by the `make` command. - ## This is required to prevent `make` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `make` command. - ## - ## `regex` can be set to true if `check` is a regular expression. - ## - ## If `make.exe` is missing and `mingw32-make.exe` is available, it will - ## be copied over to make.exe in the same location. - if findFile(check, path, regex = regex).len != 0: - return - - echo "# Running make " & flags - echo "# Path: " & path - - var - cmd = findExe("make") - - if cmd.len == 0: - cmd = findExe("mingw32-make") - if cmd.len != 0: - cpFile(cmd, cmd.replace("mingw32-make", "make")) - doAssert cmd.len != 0, "Make not found" - - cmd = &"cd {path.sanitizePath} && make" - if flags.len != 0: - cmd &= &" {flags}" - - echoDebug execAction(cmd).output - - doAssert findFile(check, path, regex = regex).len != 0, "make failed" - -proc getCompilerMode*(path: string): string = - ## Determines a target language mode from an input filename, if one is not already specified. - let file = path.splitFile() - if file.ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: - result = "cpp" - elif file.ext in [".h", ".c"]: - result = "c" - -proc getGccModeArg*(mode: string): string = - ## Produces a GCC argument that explicitly sets the language mode to be used by the compiler. - if mode == "cpp": - result = "-xc++" - elif mode == "c": - result = "-xc" - -proc getCompiler*(): string = - var - compiler = - when defined(gcc): - "gcc" - elif defined(clang): - "clang" - else: - doAssert false, "Nimterop only supports gcc and clang at this time" - - result = getEnv("CC", compiler) - -proc getGccPaths*(mode: string): seq[string] = - var - nul = when defined(Windows): "nul" else: "/dev/null" - inc = false - - (outp, _) = execAction(&"""{getCompiler()} -Wp,-v {getGccModeArg(mode)} {nul}""", die = false) - - for line in outp.splitLines(): - if "#include <...> search starts here" in line: - inc = true - continue - elif "End of search list" in line: - break - if inc: - var - path = line.strip().normalizedPath() - if path notin result: - result.add path - - when defined(osx): - result.add(execAction("xcrun --show-sdk-path").output.strip() & "/usr/include") - -proc getGccLibPaths*(mode: string): seq[string] = - var - nul = when defined(Windows): "nul" else: "/dev/null" - linker = when defined(OSX): "-Xlinker" else: "" - - (outp, _) = execAction(&"""{getCompiler()} {linker} -v {getGccModeArg(mode)} {nul}""", die = false) - - for line in outp.splitLines(): - if "LIBRARY_PATH=" in line: - for path in line[13 .. ^1].split(PathSep): - var - path = path.strip().normalizedPath() - if path notin result: - result.add path - break - elif '\t' in line: - var - path = line.strip().normalizedPath() - if path notin result: - result.add path - - when defined(osx): - result.add "/usr/lib" - -proc getGccInfo*(): tuple[arch, os, compiler, version: string] = - let - (outp, _) = execAction(&"{getCompiler()} -v") - for line in outp.splitLines(): - if line.startsWith("Target: "): - result.arch = line.split(' ')[1].split('-')[0] - result.os = - if "linux" in line: - "linux" - elif "android" in line: - "android" - elif "darwin" in line: - "macos" - elif "w64" in line or "mingw" in line: - "windows" - else: - "unknown" - elif " version " in line: - result.version = line.split(" version ")[1].split(' ')[0] - if "clang" in outp: - if result.os == "macos": - result.compiler = "apple-clang" - else: - result.compiler = "clang" - else: - result.compiler = "gcc" - template fixOutDir() {.dirty.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir @@ -881,565 +60,74 @@ proc compareVersions*(ver1, ver2: string): int = if p1 < p2: return -1 elif p1 > p2: return 1 -# Conan support -include conan - -# Julia Binary Builder support -include jbb - -proc getStdPath(header, mode: string): string = - for inc in getGccPaths(mode): - result = findFile(header, inc, recurse = false, first = true) - if result.len != 0: - break - -proc getStdLibPath(lname, mode: string): string = - for lib in getGccLibPaths(mode): - result = findFile(lname, lib, recurse = false, first = true, regex = true) - if result.len != 0: - break - -proc getGitPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No git url setup for " & header - doAssert findExe("git").len != 0, "git executable missing" - - gitPull(url, outdir, checkout = version) - - result = findFile(header, outdir) - -proc getDlPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No download url setup for " & header - - var - dlurl = url - if "$#" in url or "$1" in url: - doAssert version.len != 0, "Need version for download url" - dlurl = url % version - else: - doAssert version.len == 0, "Download url does not contain version" - - downloadUrl(dlurl, outdir) - - var - dirname = "" - for kind, path in walkDir(outdir, relative = true): - if kind == pcFile and path != dlurl.extractFilename(): - dirname = "" - break - elif kind == pcDir: - if dirname.len == 0: - dirname = path - else: - dirname = "" - break - - if dirname.len != 0: - for kind, path in walkDir(outdir / dirname, relative = true): - mvFile(outdir / dirname / path, outdir / path) - - result = findFile(header, outdir) - -proc getConanPath(header, uri, outdir, version: string, shared: bool): string = - var - uri = uri - - if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan.io uri: " & uri - uri = uri % version - elif version.len != 0: - uri = uri & "/" & version - - let - pkg = newConanPackageFromUri(uri, shared) - downloadConan(pkg, outdir) - - result = findFile(header, outdir) - -proc getConanLDeps(outdir: string): seq[string] = - let - pkg = loadConanInfo(outdir) - - result = pkg.getConanLDeps(outdir) - -proc getJBBPath(header, uri, outdir, version: string): string = - let - spl = uri.split('/', 1) - name = spl[0] - hasVersion = version.len != 0 - - var - ver = - if spl.len == 2: - spl[1] - else: - "" - - if ver.len != 0: - if "$#" in ver or "$1" in ver: - doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri - ver = ver % version - elif hasVersion: - doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & - version & "\"` for BinaryBuilder.org" - elif hasVersion: - ver = version - - let - pkg = newJBBPackage(name, ver) - downloadJBB(pkg, outdir) - - result = findFile(header, outdir) - -proc getJBBLDeps(outdir: string, shared: bool): seq[string] = - let - pkg = loadJBBInfo(outdir) - - result = pkg.getJBBLDeps(outdir, shared) - -proc getLocalPath(header, outdir: string): string = - if outdir.len != 0: - result = findFile(header, outdir) - -proc getNumProcs(): string = +proc fixCmd(cmd: string): string = when defined(Windows): - getEnv("NUMBER_OF_PROCESSORS").strip() - elif defined(linux): - execAction("nproc").output.strip() - elif defined(macosx) or defined(FreeBSD): - execAction("sysctl -n hw.ncpu").output.strip() + # Replace 'cd d:\abc' with 'd: && cd d:\abc` + var filteredCmd = cmd + if cmd.toLower().startsWith("cd"): + var + colonIndex = cmd.find(":") + driveLetter = cmd.substr(colonIndex-1, colonIndex) + if (driveLetter[0].isAlphaAscii() and + driveLetter[1] == ':' and + colonIndex == 4): + filteredCmd = &"{driveLetter} && {cmd}" + result = "cmd /c " & filteredCmd + elif defined(posix): + result = cmd else: - "1" + doAssert false -proc buildWithCmake(outdir, flags: string): BuildStatus = - if not fileExists(outdir / "Makefile"): - if fileExists(outdir / "CMakeLists.txt"): - if findExe("cmake").len != 0: - var - gen = "" - when defined(Windows): - if findExe("sh").len != 0: - let - uname = execAction("sh -c uname -a").output.toLowerAscii() - if uname.contains("msys"): - gen = "MSYS Makefiles".quoteShell - elif uname.contains("mingw"): - gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" - else: - echo "Unsupported system: " & uname - else: - gen = "MinGW Makefiles".quoteShell - else: - gen = "Unix Makefiles".quoteShell - if findExe("ccache").len != 0: - gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - result.buildPath = outdir / "buildcache" - cmake(result.buildPath, "Makefile", &".. -G {gen} {flags}") - result.built = true - else: - result.error = "cmake capable but cmake executable missing" - else: - result.buildPath = outdir +# Nim cfg file related functionality +include "."/build/nimconf -proc buildWithAutoConf(outdir, flags: string): BuildStatus = - if not fileExists(outdir / "Makefile"): - if findExe("bash").len != 0: - for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: - if fileExists(outdir / file): - configure(outdir, "Makefile", flags) - result.buildPath = outdir - result.built = true - break - else: - result.error = "configure capable but bash executable missing" - else: - result.buildPath = outdir +proc getNimteropCacheDir(): string = + # Get location to cache all nimterop artifacts + result = getNimcacheDir() / "nimterop" -proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = - var - lpath = findFile(lname, outdir, regex = true) - makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" - makePath = outdir +# Functionality shelled out to external executables +include "."/build/shell - if lpath.len != 0: - return lpath - - var buildStatus: BuildStatus - - for buildType in buildTypes: - case buildType - of btCmake: - buildStatus = buildWithCmake(makePath, cmakeFlags) - of btAutoconf: - buildStatus = buildWithAutoConf(makePath, conFlags) - - if buildStatus.built: - break - - if buildStatus.buildPath.len > 0: - let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 - - if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): - make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) - buildStatus.built = true - - let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir - doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" - - result = findFile(lname, outdir, regex = true) - -proc getDynlibExt(): string = - when defined(Windows): - result = "[0-9.\\-]*\\.dll" - elif defined(linux) or defined(FreeBSD): - result = "\\.so[0-9.]*" - elif defined(macosx): - result = "[0-9.\\-]*\\.dylib" - -var - gDefines {.compileTime.} = initTable[string, string]() - -macro setDefines*(defs: static openArray[string]): untyped = - ## Specify `-d:xxx` values in code instead of having to rely on the command - ## line or `cfg` or `nims` files. +proc getProjectCacheDir*(name: string, forceClean = true): string = + ## Get a cache directory where all nimterop artifacts can be stored ## - ## At this time, Nim does not allow creation of `-d:xxx` defines in code. In - ## addition, Nim only loads config files for the module being compiled but not - ## for imported packages. This becomes a challenge when wanting to ship a wrapper - ## library that wants to control `getHeader()` for an underlying package. + ## Projects can use this location to download source code and build binaries + ## that can be then accessed by multiple apps. This is created under the + ## per-user Nim cache directory. ## - ## E.g. nimarchive wanting to set `-d:lzmaStatic` + ## Use `name` to specify the subdirectory name for a project. ## - ## The consumer of nimarchive would need to set such defines as part of their - ## project, making it inconvenient. + ## `forceClean` is enabled by default and effectively deletes the folder + ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows + ## any project to start out with a clean cache dir on a forced build. ## - ## By calling this proc with the defines preferred before importing such a module, - ## the caller can set the behavior in code instead. + ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same + ## `name` when `forceClean = true` else checked out source might get deleted + ## at the wrong time during build. ## - ## .. code-block:: nim + ## E.g. + ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` ## - ## setDefines(@["lzmaStatic", "lzmaDL", "lzmaSetVer=5.2.4"]) - ## - ## import lzma - for def in defs: - let - nv = def.strip().split("=", maxsplit = 1) - if nv.len != 0: - let - n = nv[0] - v = - if nv.len == 2: - nv[1] - else: - "" - gDefines[n] = v + ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so + ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. + result = getNimteropCacheDir() / name -macro clearDefines*(): untyped = - ## Clear all defines set using `setDefines()`. - gDefines.clear() + if forceClean and compileOption("forceBuild"): + echo "# Removing " & result + rmDir(result) -macro isDefined*(def: untyped): untyped = - ## Check if `-d:xxx` is set globally or via `setDefines()` - let - sdef = gDefines.hasKey(def.strVal()) - result = newNimNode(nnkStmtList) - result.add(quote do: - when defined(`def`) or `sdef` != 0: - true - else: - false - ) +# C compiler support +include "."/build/ccompiler -macro getHeader*( - header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", jbburi: static[string] = "", - outdir: static[string] = "", libdir: static[string] = "", - conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = - ## Get the path to a header file for wrapping with - ## `cImport() `_ or - ## `c2nImport() `_. - ## - ## This proc checks `-d:xxx` defines based on the header name (e.g. lzma from lzma.h), - ## and accordingly employs different ways to obtain the source. - ## - ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux - ## `-d:xxxGit` - clone source from a git repo specified in `giturl` - ## `-d:xxxDL` - download source from `dlurl` and extract if required - ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with - ## format `pkgname[/version[@user/channel][:bhash]]` - ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with - ## format `pkgname[/version]` - ## - ## This allows a single wrapper to be used in different ways depending on the user's needs. - ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. - ## The user can opt to download the sources to `outdir` using any other method such as - ## git sub-modules, vendoring or pointing to a repository that was already cloned. - ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, - ## `Conan` or `JBB`. This allows using a system installed library if available before - ## falling back to manual building. The user would need to specify both `-d:xxxStd` and - ## one of the other methods. - ## - ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL - ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for - ## `Conan` and `JBB` if available. If no version is specified, the latest release of the - ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional - ## URI information: - ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` - ## - ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` - ## filename is used instead. - ## - ## All defines can also be set in code using `setDefines()` and checked for using - ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. - ## - ## The library is then configured (with `cmake` or `autotools` if possible) and built - ## using `make`, unless using `-d:xxxStd` which presumes that the system package - ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` - ## or `-d:xxxJBB` which download pre-built binaries. - ## - ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call - ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency - ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. - ## - ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their - ## dependencies to that directory. This prevents any runtime failures if `outdir` gets - ## removed or its contents changed. By default, `libdir` is set to the output directory - ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will - ## reflect this new location. `libdir` is ignored for `Std` mode. - ## - ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. Note - ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the - ## `xxxLPath` will include paths to all of them separated by space in the right order for - ## linking. - ## - ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of - ## OSX will complain if statically linking to these binaries. Further, all Conan binaries - ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc - ## or clang might lead to incompatibility issues if the library uses Visual Studio - ## specific compiler features. - ## - ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, - ## `cmake` and `make` in case additional configuration is required as part of the build - ## process. - ## - ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for - ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is - ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma - ## separate for multiple alternate names without spaces. - ## - ## The original header name is not included by default if `altNames` is set since it could - ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if - ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when - ## required. - ## - ## `buildTypes` specifies a list of ordered build strategies to use when building the - ## downloaded source files. Default is [btCmake, btAutoconf] - ## - ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or - ## downloaded but before the library is built. This might be needed if some initial prep - ## needs to be done before compilation. A few values are provided to the hook to help - ## provide context: - ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path - ## in the downloaded source code. - ## - ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get - ## called prior to the build process. - var - origname = header.extractFilename().split(".")[0] - name = origname.split(seps = AllChars-Letters-Digits).join() +when not defined(TOAST): + # configure, cmake, make support + include "."/build/tools - # Default to origname if not specified - conanuri = if conanuri.len != 0: conanuri else: origname - jbburi = if jbburi.len != 0: jbburi else: origname + # Conan.io support + include "."/build/conan - # -d:xxx for this header - stdStr = name & "Std" - gitStr = name & "Git" - dlStr = name & "DL" - conanStr = name & "Conan" - jbbStr = name & "JBB" + # Julia BinaryBuilder.org support + include "."/build/jbb - staticStr = name & "Static" - verStr = name & "SetVer" - - # Ident nodes of the -d:xxx to check in when statements - nameStd = newIdentNode(stdStr) - nameGit = newIdentNode(gitStr) - nameDL = newIdentNode(dlStr) - nameConan = newIdentNode(conanStr) - nameJBB = newIdentNode(jbbStr) - - nameStatic = newIdentNode(staticStr) - - # Consts to generate - path = newIdentNode(name & "Path") - lpath = newIdentNode(name & "LPath") - ldeps = newIdentNode(name & "LDeps") - version = newIdentNode(verStr) - lname = newIdentNode(name & "LName") - preBuild = newIdentNode(name & "PreBuild") - - # Regex for library search - lre = "(lib)?$1[_-]?(static)?" - - # If -d:xxx set with setDefines() - stdVal = gDefines.hasKey(stdStr) - gitVal = gDefines.hasKey(gitStr) - dlVal = gDefines.hasKey(dlStr) - conanVal = gDefines.hasKey(conanStr) - jbbVal = gDefines.hasKey(jbbStr) - staticVal = gDefines.hasKey(staticStr) - verVal = - if gDefines.hasKey(verStr): - gDefines[verStr] - else: - "" - mode = getCompilerMode(header) - - libdir = if libdir.len != 0: libdir else: getOutDir() - - # Use alternate library names if specified for regex search - if altNames.len != 0: - lre = lre % ("(" & altNames.replace(",", "|") & ")") - else: - lre = lre % origname - - result = newNimNode(nnkStmtList) - result.add(quote do: - # Need to check -d:xxx or setDefines() - const - `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 - `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 - `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 - `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 - `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 - `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 - - # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = - when `nameGit`: - getGitPath(header, giturl, outdir, version) - elif `nameDL`: - getDlPath(header, dlurl, outdir, version) - elif `nameConan`: - getConanPath(header, conanuri, outdir, version, shared) - elif `nameJBB`: - getJBBPath(header, jbburi, outdir, version) - else: - getLocalPath(header, outdir) - - const - `version`* {.strdefine.} = `verVal` - `lname` = - when `nameStatic`: - `lre` & "\\.(a|lib)" - else: - `lre` & getDynlibExt() - - # Look in standard path if requested by user - stdPath = - when `nameStd`: getStdPath(`header`, `mode`) else: "" - stdLPath = - when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" - - useStd = stdPath.len != 0 and stdLPath.len != 0 - - # Look elsewhere if requested while prioritizing standard paths - prePath = - when useStd: - stdPath - else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - - # Run preBuild hook before building library if not Std, Conan or JBB - when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): - static: - `preBuild`(`outdir`, prePath) - - let - # Library binary path - build if not standard / conan / jbb - lpath {.compileTime.} = - when useStd: - stdLPath - elif `nameConan` or `nameJBB`: - findFile(`lname`, `outdir`, regex = true) - else: - buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) - - # Library dependecy paths - ldeps {.compileTime.}: seq[string] = - when not useStd: - when `nameConan`: - getConanLDeps(`outdir`) - elif `nameJBB`: - getJBBLDeps(`outdir`, not `nameStatic`) - else: - @[] - else: - @[] - - const - # Header path - search again in case header is generated in build - `path`* = - if prePath.len != 0: - prePath - else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - - static: - doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & - "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` - doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" - - when `nameStatic`: - const - `lpath`* = lpath - `ldeps`* = ldeps - - # Automatically link with static library and dependencies - {.passL: `lpath`.} - if `ldeps`.len != 0: - {.passL: `ldeps`.join(" ").} - - static: - echo "# Including library " & lpath - if `ldeps`.len != 0: - echo "# Including dependencies " & `ldeps`.join(" ") - else: - const - `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath - `ldeps`* = - when not useStd: - block: - var - ldeps = ldeps - copied: seq[string] - for i in 0 ..< ldeps.len: - let - lname = ldeps[i].extractFilename() - ldeptgt = `libdir` / lname - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): - cpFile(ldeps[i], ldeptgt, psymlink = true) - copied.add lname - ldeps[i] = ldeptgt - # Copy downloaded dependencies to `libdir` - if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` - ldeps - else: - ldeps - - static: - when not useStd: - # Copy downloaded shared libraries to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` - cpFile(lpath, `lpath`) - - echo "# Including library " & `lpath` - ) + # getHeader support + include "."/build/getheader diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim new file mode 100644 index 0000000..c33c94a --- /dev/null +++ b/nimterop/build/ccompiler.nim @@ -0,0 +1,101 @@ +import os, strformat, strutils + +proc getCompilerMode*(path: string): string = + ## Determines a target language mode from an input filename, if one is not already specified. + let file = path.splitFile() + if file.ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: + result = "cpp" + elif file.ext in [".h", ".c"]: + result = "c" + +proc getGccModeArg*(mode: string): string = + ## Produces a GCC argument that explicitly sets the language mode to be used by the compiler. + if mode == "cpp": + result = "-xc++" + elif mode == "c": + result = "-xc" + +proc getCompiler*(): string = + var + compiler = + when defined(gcc): + "gcc" + elif defined(clang): + "clang" + else: + doAssert false, "Nimterop only supports gcc and clang at this time" + + result = getEnv("CC", compiler) + +proc getGccPaths*(mode: string): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + inc = false + + (outp, _) = execAction(&"""{getCompiler()} -Wp,-v {getGccModeArg(mode)} {nul}""", die = false) + + for line in outp.splitLines(): + if "#include <...> search starts here" in line: + inc = true + continue + elif "End of search list" in line: + break + if inc: + var + path = line.strip().normalizedPath() + if path notin result: + result.add path + + when defined(osx): + result.add(execAction("xcrun --show-sdk-path").output.strip() & "/usr/include") + +proc getGccLibPaths*(mode: string): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + linker = when defined(OSX): "-Xlinker" else: "" + + (outp, _) = execAction(&"""{getCompiler()} {linker} -v {getGccModeArg(mode)} {nul}""", die = false) + + for line in outp.splitLines(): + if "LIBRARY_PATH=" in line: + for path in line[13 .. ^1].split(PathSep): + var + path = path.strip().normalizedPath() + if path notin result: + result.add path + break + elif '\t' in line: + var + path = line.strip().normalizedPath() + if path notin result: + result.add path + + when defined(osx): + result.add "/usr/lib" + +proc getGccInfo*(): tuple[arch, os, compiler, version: string] = + let + (outp, _) = execAction(&"{getCompiler()} -v") + for line in outp.splitLines(): + if line.startsWith("Target: "): + result.arch = line.split(' ')[1].split('-')[0] + result.os = + if "linux" in line: + "linux" + elif "android" in line: + "android" + elif "darwin" in line: + "macos" + elif "w64" in line or "mingw" in line: + "windows" + else: + "unknown" + elif " version " in line: + result.version = line.split(" version ")[1].split(' ')[0] + if "clang" in outp: + if result.os == "macos": + result.compiler = "apple-clang" + else: + result.compiler = "clang" + else: + result.compiler = "gcc" diff --git a/nimterop/conan.nim b/nimterop/build/conan.nim similarity index 100% rename from nimterop/conan.nim rename to nimterop/build/conan.nim diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim new file mode 100644 index 0000000..3852fbf --- /dev/null +++ b/nimterop/build/getheader.nim @@ -0,0 +1,504 @@ +import macros, os, strutils, tables + +var + gDefines {.compileTime.} = initTable[string, string]() + +macro setDefines*(defs: static openArray[string]): untyped = + ## Specify `-d:xxx` values in code instead of having to rely on the command + ## line or `cfg` or `nims` files. + ## + ## At this time, Nim does not allow creation of `-d:xxx` defines in code. In + ## addition, Nim only loads config files for the module being compiled but not + ## for imported packages. This becomes a challenge when wanting to ship a wrapper + ## library that wants to control `getHeader()` for an underlying package. + ## + ## E.g. nimarchive wanting to set `-d:lzmaStatic` + ## + ## The consumer of nimarchive would need to set such defines as part of their + ## project, making it inconvenient. + ## + ## By calling this proc with the defines preferred before importing such a module, + ## the caller can set the behavior in code instead. + ## + ## .. code-block:: nim + ## + ## setDefines(@["lzmaStatic", "lzmaDL", "lzmaSetVer=5.2.4"]) + ## + ## import lzma + for def in defs: + let + nv = def.strip().split("=", maxsplit = 1) + if nv.len != 0: + let + n = nv[0] + v = + if nv.len == 2: + nv[1] + else: + "" + gDefines[n] = v + +macro clearDefines*(): untyped = + ## Clear all defines set using `setDefines()`. + gDefines.clear() + +macro isDefined*(def: untyped): untyped = + ## Check if `-d:xxx` is set globally or via `setDefines()` + let + sdef = gDefines.hasKey(def.strVal()) + result = newNimNode(nnkStmtList) + result.add(quote do: + when defined(`def`) or `sdef` != 0: + true + else: + false + ) + +proc getDynlibExt(): string = + when defined(Windows): + result = "[0-9.\\-]*\\.dll" + elif defined(linux) or defined(FreeBSD): + result = "\\.so[0-9.]*" + elif defined(macosx): + result = "[0-9.\\-]*\\.dylib" + +proc getStdPath(header, mode: string): string = + for inc in getGccPaths(mode): + result = findFile(header, inc, recurse = false, first = true) + if result.len != 0: + break + +proc getStdLibPath(lname, mode: string): string = + for lib in getGccLibPaths(mode): + result = findFile(lname, lib, recurse = false, first = true, regex = true) + if result.len != 0: + break + +proc getGitPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No git url setup for " & header + doAssert findExe("git").len != 0, "git executable missing" + + gitPull(url, outdir, checkout = version) + + result = findFile(header, outdir) + +proc getDlPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No download url setup for " & header + + var + dlurl = url + if "$#" in url or "$1" in url: + doAssert version.len != 0, "Need version for download url" + dlurl = url % version + else: + doAssert version.len == 0, "Download url does not contain version" + + downloadUrl(dlurl, outdir) + + var + dirname = "" + for kind, path in walkDir(outdir, relative = true): + if kind == pcFile and path != dlurl.extractFilename(): + dirname = "" + break + elif kind == pcDir: + if dirname.len == 0: + dirname = path + else: + dirname = "" + break + + if dirname.len != 0: + for kind, path in walkDir(outdir / dirname, relative = true): + mvFile(outdir / dirname / path, outdir / path) + + result = findFile(header, outdir) + +proc getConanPath(header, uri, outdir, version: string, shared: bool): string = + var + uri = uri + + if "$#" in uri or "$1" in uri: + doAssert version.len != 0, "Need version for Conan.io uri: " & uri + uri = uri % version + elif version.len != 0: + uri = uri & "/" & version + + let + pkg = newConanPackageFromUri(uri, shared) + downloadConan(pkg, outdir) + + result = findFile(header, outdir) + +proc getConanLDeps(outdir: string): seq[string] = + let + pkg = loadConanInfo(outdir) + + result = pkg.getConanLDeps(outdir) + +proc getJBBPath(header, uri, outdir, version: string): string = + let + spl = uri.split('/', 1) + name = spl[0] + hasVersion = version.len != 0 + + var + ver = + if spl.len == 2: + spl[1] + else: + "" + + if ver.len != 0: + if "$#" in ver or "$1" in ver: + doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri + ver = ver % version + elif hasVersion: + doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & + version & "\"` for BinaryBuilder.org" + elif hasVersion: + ver = version + + let + pkg = newJBBPackage(name, ver) + downloadJBB(pkg, outdir) + + result = findFile(header, outdir) + +proc getJBBLDeps(outdir: string, shared: bool): seq[string] = + let + pkg = loadJBBInfo(outdir) + + result = pkg.getJBBLDeps(outdir, shared) + +proc getLocalPath(header, outdir: string): string = + if outdir.len != 0: + result = findFile(header, outdir) + +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = + var + lpath = findFile(lname, outdir, regex = true) + makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" + makePath = outdir + + if lpath.len != 0: + return lpath + + var buildStatus: BuildStatus + + for buildType in buildTypes: + case buildType + of btCmake: + buildStatus = buildWithCmake(makePath, cmakeFlags) + of btAutoconf: + buildStatus = buildWithAutoConf(makePath, conFlags) + + if buildStatus.built: + break + + if buildStatus.buildPath.len > 0: + let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 + + if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): + make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) + buildStatus.built = true + + let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir + doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" + + result = findFile(lname, outdir, regex = true) + +macro getHeader*( + header: static[string], giturl: static[string] = "", dlurl: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", + outdir: static[string] = "", libdir: static[string] = "", + conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", + altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = + ## Get the path to a header file for wrapping with + ## `cImport() `_ or + ## `c2nImport() `_. + ## + ## This proc checks `-d:xxx` defines based on the header name (e.g. lzma from lzma.h), + ## and accordingly employs different ways to obtain the source. + ## + ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux + ## `-d:xxxGit` - clone source from a git repo specified in `giturl` + ## `-d:xxxDL` - download source from `dlurl` and extract if required + ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with + ## format `pkgname[/version[@user/channel][:bhash]]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with + ## format `pkgname[/version]` + ## + ## This allows a single wrapper to be used in different ways depending on the user's needs. + ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. + ## The user can opt to download the sources to `outdir` using any other method such as + ## git sub-modules, vendoring or pointing to a repository that was already cloned. + ## + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, + ## `Conan` or `JBB`. This allows using a system installed library if available before + ## falling back to manual building. The user would need to specify both `-d:xxxStd` and + ## one of the other methods. + ## + ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag + ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL + ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for + ## `Conan` and `JBB` if available. If no version is specified, the latest release of the + ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional + ## URI information: + ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` + ## + ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` + ## filename is used instead. + ## + ## All defines can also be set in code using `setDefines()` and checked for using + ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. + ## + ## The library is then configured (with `cmake` or `autotools` if possible) and built + ## using `make`, unless using `-d:xxxStd` which presumes that the system package + ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` + ## or `-d:xxxJBB` which download pre-built binaries. + ## + ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call + ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can + ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. + ## + ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their + ## dependencies to that directory. This prevents any runtime failures if `outdir` gets + ## removed or its contents changed. By default, `libdir` is set to the output directory + ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will + ## reflect this new location. `libdir` is ignored for `Std` mode. + ## + ## `-d:xxxStatic` can be specified to statically link with the library instead. This + ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the + ## `xxxLPath` will include paths to all of them separated by space in the right order for + ## linking. + ## + ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of + ## OSX will complain if statically linking to these binaries. Further, all Conan binaries + ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc + ## or clang might lead to incompatibility issues if the library uses Visual Studio + ## specific compiler features. + ## + ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, + ## `cmake` and `make` in case additional configuration is required as part of the build + ## process. + ## + ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for + ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is + ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma + ## separate for multiple alternate names without spaces. + ## + ## The original header name is not included by default if `altNames` is set since it could + ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if + ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when + ## required. + ## + ## `buildTypes` specifies a list of ordered build strategies to use when building the + ## downloaded source files. Default is [btCmake, btAutoconf] + ## + ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or + ## downloaded but before the library is built. This might be needed if some initial prep + ## needs to be done before compilation. A few values are provided to the hook to help + ## provide context: + ## + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path + ## in the downloaded source code. + ## + ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get + ## called prior to the build process. + var + origname = header.extractFilename().split(".")[0] + name = origname.split(seps = AllChars-Letters-Digits).join() + + # Default to origname if not specified + conanuri = if conanuri.len != 0: conanuri else: origname + jbburi = if jbburi.len != 0: jbburi else: origname + + # -d:xxx for this header + stdStr = name & "Std" + gitStr = name & "Git" + dlStr = name & "DL" + conanStr = name & "Conan" + jbbStr = name & "JBB" + + staticStr = name & "Static" + verStr = name & "SetVer" + + # Ident nodes of the -d:xxx to check in when statements + nameStd = newIdentNode(stdStr) + nameGit = newIdentNode(gitStr) + nameDL = newIdentNode(dlStr) + nameConan = newIdentNode(conanStr) + nameJBB = newIdentNode(jbbStr) + + nameStatic = newIdentNode(staticStr) + + # Consts to generate + path = newIdentNode(name & "Path") + lpath = newIdentNode(name & "LPath") + ldeps = newIdentNode(name & "LDeps") + version = newIdentNode(verStr) + lname = newIdentNode(name & "LName") + preBuild = newIdentNode(name & "PreBuild") + + # Regex for library search + lre = "(lib)?$1[_-]?(static)?" + + # If -d:xxx set with setDefines() + stdVal = gDefines.hasKey(stdStr) + gitVal = gDefines.hasKey(gitStr) + dlVal = gDefines.hasKey(dlStr) + conanVal = gDefines.hasKey(conanStr) + jbbVal = gDefines.hasKey(jbbStr) + staticVal = gDefines.hasKey(staticStr) + verVal = + if gDefines.hasKey(verStr): + gDefines[verStr] + else: + "" + mode = getCompilerMode(header) + + libdir = if libdir.len != 0: libdir else: getOutDir() + + # Use alternate library names if specified for regex search + if altNames.len != 0: + lre = lre % ("(" & altNames.replace(",", "|") & ")") + else: + lre = lre % origname + + result = newNimNode(nnkStmtList) + result.add(quote do: + # Need to check -d:xxx or setDefines() + const + `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 + `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 + `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 + `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 + `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 + `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 + + # Search for header in outdir (after retrieving code) depending on -d:xxx mode + proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = + when `nameGit`: + getGitPath(header, giturl, outdir, version) + elif `nameDL`: + getDlPath(header, dlurl, outdir, version) + elif `nameConan`: + getConanPath(header, conanuri, outdir, version, shared) + elif `nameJBB`: + getJBBPath(header, jbburi, outdir, version) + else: + getLocalPath(header, outdir) + + const + `version`* {.strdefine.} = `verVal` + `lname` = + when `nameStatic`: + `lre` & "\\.(a|lib)" + else: + `lre` & getDynlibExt() + + # Look in standard path if requested by user + stdPath = + when `nameStd`: getStdPath(`header`, `mode`) else: "" + stdLPath = + when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" + + useStd = stdPath.len != 0 and stdLPath.len != 0 + + # Look elsewhere if requested while prioritizing standard paths + prePath = + when useStd: + stdPath + else: + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + + # Run preBuild hook before building library if not Std, Conan or JBB + when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): + static: + `preBuild`(`outdir`, prePath) + + let + # Library binary path - build if not standard / conan / jbb + lpath {.compileTime.} = + when useStd: + stdLPath + elif `nameConan` or `nameJBB`: + findFile(`lname`, `outdir`, regex = true) + else: + buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) + + # Library dependecy paths + ldeps {.compileTime.}: seq[string] = + when not useStd: + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] + else: + @[] + + const + # Header path - search again in case header is generated in build + `path`* = + if prePath.len != 0: + prePath + else: + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + + static: + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` + doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" + + when `nameStatic`: + const + `lpath`* = lpath + `ldeps`* = ldeps + + # Automatically link with static library and dependencies + {.passL: `lpath`.} + if `ldeps`.len != 0: + {.passL: `ldeps`.join(" ").} + + static: + echo "# Including library " & lpath + if `ldeps`.len != 0: + echo "# Including dependencies " & `ldeps`.join(" ") + else: + const + `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath + `ldeps`* = + when not useStd: + block: + var + ldeps = ldeps + copied: seq[string] + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilename() + ldeptgt = `libdir` / lname + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): + cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname + ldeps[i] = ldeptgt + # Copy downloaded dependencies to `libdir` + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + ldeps + else: + ldeps + + static: + when not useStd: + # Copy downloaded shared libraries to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + cpFile(lpath, `lpath`) + + echo "# Including library " & `lpath` + ) diff --git a/nimterop/jbb.nim b/nimterop/build/jbb.nim similarity index 99% rename from nimterop/jbb.nim rename to nimterop/build/jbb.nim index c282197..15bc578 100644 --- a/nimterop/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,4 +1,4 @@ -import json, os, strutils +import json, os, strformat, strutils, tables when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal diff --git a/nimterop/nimconf.nim b/nimterop/build/nimconf.nim similarity index 100% rename from nimterop/nimconf.nim rename to nimterop/build/nimconf.nim diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim new file mode 100644 index 0000000..4f3c316 --- /dev/null +++ b/nimterop/build/shell.nim @@ -0,0 +1,465 @@ +import os, strformat, strutils + +proc sleep*(milsecs: int) = + ## Sleep at compile time + let + cmd = + when defined(Windows): + "cmd /c timeout " + else: + "sleep " + + discard gorgeEx(cmd & $(milsecs / 1000)) + +proc execAction*(cmd: string, retry = 0, die = true, cache = false, + cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = + ## Execute an external command - supported at compile time + ## + ## Checks if command exits successfully before returning. If not, an + ## error is raised. Always caches results to be used in nimsuggest or nimcheck + ## mode. + ## + ## `retry` - number of times command should be retried before error + ## `die = false` - return on errors + ## `cache = true` - cache results unless cleared with -f + ## `cacheKey` - key to create unique cache entry + let + ccmd = fixCmd(cmd) + + when nimvm: + # Cache results for speedup if cache = true + # Else cache for preserving functionality in nimsuggest and nimcheck + let + hash = (ccmd & cacheKey).hash().abs() + cachePath = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash + cacheFile = cachePath & ".txt" + retFile = cachePath & "_ret.txt" + + when defined(nimsuggest) or defined(nimcheck): + # Load results from cache file if generated in previous run + if fileExists(cacheFile) and fileExists(retFile): + result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() + elif die: + doAssert false, "Results not cached - run nim c/cpp at least once\n" & ccmd + else: + if cache and fileExists(cacheFile) and fileExists(retFile) and not compileOption("forceBuild"): + # Return from cache when requested + result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() + else: + # Execute command and store results in cache + (result.output, result.ret) = gorgeEx(ccmd) + if result.ret == 0 or die == false: + # mkdir for execCache dir (circular dependency) + let dir = cacheFile.parentDir() + if not dirExists(dir): + let flag = when not defined(Windows): "-p" else: "" + discard execAction(&"mkdir {flag} {dir.sanitizePath}") + cacheFile.writeFile(result.output) + retFile.writeFile($result.ret) + else: + # Used by toast + (result.output, result.ret) = execCmdEx(ccmd) + + # On failure, retry or die as requested + if result.ret != 0: + if retry > 0: + if not onRetry.isNil: + onRetry() + sleep(500) + result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) + elif die: + doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & + "\nresult:\n" & result.output + +proc findExe*(exe: string): string = + ## Find the specified executable using the `which`/`where` command - supported + ## at compile time + var + cmd = + when defined(Windows): + "where " & exe + else: + "which " & exe + + (output, ret) = execAction(cmd, die = false) + + if ret == 0: + return output.splitLines()[0].strip() + +proc mkDir*(dir: string) = + ## Create a directory at compile time + ## + ## The `os` module is not available at compile time so a few + ## crucial helper functions are included with nimterop. + if not dirExists(dir): + let + flag = when not defined(Windows): "-p" else: "" + discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) + +proc cpFile*(source, dest: string, psymlink = false, move = false) = + ## Copy a file from `source` to `dest` at compile time + ## + ## `psymlink = true` preserves symlinks instead of dereferencing on posix + let + source = source.replace("/", $DirSep) + dest = dest.replace("/", $DirSep) + cmd = + when defined(Windows): + if move: + "move /y" + else: + "copy /y" + else: + if move: + "mv -f" + else: + if psymlink: + "cp -fa" + else: + "cp -f" + + discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) + +proc mvFile*(source, dest: string) = + ## Move a file from `source` to `dest` at compile time + cpFile(source, dest, move=true) + +proc rmFile*(source: string, dir = false) = + ## Remove a file or pattern at compile time + let + source = source.replace("/", $DirSep) + cmd = + when defined(Windows): + if dir: + "rd /s/q" + else: + "del /s/q/f" + else: + "rm -rf" + exists = + if dir: + dirExists(source) + else: + fileExists(source) + + if exists: + discard execAction(&"{cmd} {source.sanitizePath}", retry = 2) + +proc rmDir*(dir: string) = + ## Remove a directory or pattern at compile time + rmFile(dir, dir = true) + +proc cleanDir*(dir: string) = + ## Remove all contents of a directory at compile time + for kind, path in walkDir(dir): + if kind == pcDir: + rmDir(path) + else: + rmFile(path) + +proc cpTree*(source, dest: string, move = false) = + ## Copy contents of source dir to the destination, not the directory itself + for kind, path in walkDir(source, relative = true): + if kind == pcDir: + cpTree(source / path, dest / path, move) + if move: + rmDir(source / path) + else: + if not dirExists(dest): + mkDir(dest) + if move: + mvFile(source / path, dest / path) + else: + cpFile(source / path, dest / path) + +proc mvTree*(source, dest: string) = + ## Move contents of source dir to the destination, not the directory itself + cpTree(source, dest, move = true) + +proc getFileDate*(fullpath: string): string = + ## Get file date for `fullpath` + var + ret = 0 + cmd = + when defined(Windows): + let + (head, tail) = fullpath.splitPath() + &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" + elif defined(Linux): + &"stat -c %y {fullpath.sanitizePath}" + elif defined(OSX) or defined(FreeBSD): + &"stat -f %m {fullpath.sanitizePath}" + + (result, ret) = execAction(cmd) + +proc touchFile*(fullpath: string) = + ## Touch file to update modified date + var + cmd = + when defined(Windows): + &"cmd /c copy /b {fullpath.sanitizePath}+" + else: + &"touch {fullpath.sanitizePath}" + + discard execAction(cmd) + +proc extractZip*(zipfile, outdir: string, quiet = false) = + ## Extract a zip file using `powershell` on Windows and `unzip` on other + ## systems to the specified output directory + var cmd = "unzip -o $#" + if defined(Windows): + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & + "'System.IO.Compression.FileSystem'; " & + "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + + if not quiet: + echo "# Extracting " & zipfile + discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") + +proc extractTar*(tarfile, outdir: string, quiet = false) = + ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory + var + cmd = "" + name = "" + + if findExe("tar").len != 0: + let + ext = tarfile.splitFile().ext.toLowerAscii() + typ = + case ext + of ".gz", ".tgz": "z" + of ".xz": "J" + of ".bz2": "j" + else: "" + + cmd = "tar xvf" & typ & " " & tarfile.sanitizePath + else: + for i in ["7z", "7za"]: + if findExe(i).len != 0: + cmd = i & " x $#" % tarfile.sanitizePath + + name = tarfile.splitFile().name + if ".tar" in name.toLowerAscii(): + cmd &= " && " & i & " x $#" % name.sanitizePath + + break + + doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath + + if not quiet: + echo "# Extracting " & tarfile + discard execAction(&"cd {outdir.sanitizePath} && {cmd}") + if name.len != 0: + rmFile(outdir / name) + +proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = + ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory + ## + ## If an archive file, it is automatically extracted after download. + let + file = url.extractFilename() + filePath = outdir / file + ext = file.splitFile().ext.toLowerAscii() + archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] + + if not (ext in archives and fileExists(filePath)): + if not quiet: + echo "# Downloading " & file + mkDir(outdir) + var cmd = findExe("curl") + if cmd.len != 0: + cmd &= " -Lk $# -o $#" + else: + cmd = findExe("wget") + if cmd.len != 0: + cmd &= " $# -O $#" + elif defined(Windows): + cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" + else: + doAssert false, "No download tool available - curl, wget" + discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, + onRetry = proc() = rmFile(filePath)) + + if ext == ".zip": + extractZip(file, outdir, quiet) + elif ext in archives: + extractTar(file, outdir, quiet) + +proc gitReset*(outdir: string) = + ## Hard reset the git repository at the specified directory + echo "# Resetting " & outdir + + let cmd = &"cd {outdir.sanitizePath} && git reset --hard" + while execAction(cmd).output.contains("Permission denied"): + sleep(1000) + echo "# Retrying ..." + +proc gitCheckout*(file, outdir: string) = + ## Checkout the specified `file` in the git repository at `outdir` + ## + ## This effectively resets all changes in the file and can be + ## used to undo any changes that were made to source files to enable + ## successful wrapping with `cImport()` or `c2nImport()`. + echo "# Resetting " & file + let file2 = file.relativePath outdir + let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" + while execAction(cmd).output.contains("Permission denied"): + sleep(500) + echo "# Retrying ..." + +proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = + ## Pull the specified git repository to the output directory + ## + ## `plist` is the list of specific files and directories or wildcards + ## to sparsely checkout. Multiple values can be specified one entry per + ## line. It is optional and if omitted, the entire repository will be + ## checked out. + ## + ## `checkout` is the git tag, branch or commit hash to checkout once + ## the repository is downloaded. This allows for pinning to a specific + ## version of the code. + if dirExists(outdir/".git"): + gitReset(outdir) + return + + let + outdirQ = outdir.sanitizePath + + mkDir(outdir) + + if not quiet: + echo "# Setting up Git repo: " & url + discard execAction(&"cd {outdirQ} && git init .") + discard execAction(&"cd {outdirQ} && git remote add origin {url}") + + if plist.len != 0: + # If a specific list of files is required, create a sparse checkout + # file for git in its config directory + let sparsefile = outdir / ".git/info/sparse-checkout" + + discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") + writeFile(sparsefile, plist) + + # In case directory has old files from another run + discard execAction(&"cd {outdirQ} && git clean -fxd") + + if checkout.len != 0: + if not quiet: + echo "# Checking out " & checkout + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) + discard execAction(&"cd {outdirQ} && git checkout {checkout}") + else: + if not quiet: + echo "# Pulling repository" + discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) + +proc gitTags*(outdir: string): seq[string] = + ## Get all the git tags in the specified directory + let + cmd = &"cd {outdir.sanitizePath} && git tag" + tags = execAction(cmd).output.splitLines() + for tag in tags: + let + tag = tag.strip() + if tag.len != 0: + result.add tag + +proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = + ## Find all matching files in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` + var + cmd = + when defined(Windows): + "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" + elif defined(linux): + "find $3 $1 -regextype egrep -regex $2" + elif defined(osx) or defined(FreeBSD): + "find -E $3 $1 -regex $2" + + recursive = "" + + if recurse: + when defined(Windows): + recursive = "--recursive" + else: + when not defined(Windows): + recursive = "-maxdepth 1" + + var + dir = dir + file = file + if not recurse: + let + pdir = file.parentDir() + if pdir.len != 0: + dir = dir / pdir + + file = file.extractFilename + + cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] + + let + (files, ret) = execAction(cmd, die = false) + if ret == 0: + for line in files.splitLines(): + let f = + when defined(Windows): + if ": " in line: + line.split(": ", maxsplit = 1)[1] + else: + "" + else: + line + if f.len != 0: + result.add f + +proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = + ## Find the file in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. + let + matches = findFiles(file, dir, recurse, regex) + for match in matches: + if (result.len == 0 or result.len > match.len): + result = match + if first: break + +proc linkLibs*(names: openArray[string], staticLink = true): string = + ## Create linker flags for specified libraries using pkg-config + ## + ## Prepends `lib` to the name so you only need `ssl` for `libssl`. + var + stat = if staticLink: "--static" else: "" + resSet: OrderedSet[string] + resSet.init() + + for name in names: + let + cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" + (libs, _) = execAction(cmd, die = false) + for lib in libs.split(" "): + resSet.incl lib + + if staticLink: + resSet.incl "--static" + + for res in resSet: + result &= " " & res + +proc getNumProcs(): string = + when defined(Windows): + getEnv("NUMBER_OF_PROCESSORS").strip() + elif defined(linux): + execAction("nproc").output.strip() + elif defined(macosx) or defined(FreeBSD): + execAction("sysctl -n hw.ncpu").output.strip() + else: + "1" diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim new file mode 100644 index 0000000..772c48b --- /dev/null +++ b/nimterop/build/tools.nim @@ -0,0 +1,260 @@ +import os, strformat, strutils + +type + BuildType* = enum + btAutoconf, btCmake + + BuildStatus = object + built: bool + buildPath: string + error: string + +proc configure*(path, check: string, flags = "") = + ## Run the GNU `configure` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## If a `configure` script is not present and an `autogen.sh` script + ## is present, it will be run before attempting `configure`. + ## + ## Next, if `configure.ac` or `configure.in` exist, `autoreconf` will + ## be executed. + ## + ## `check` is a file that will be generated by the `configure` command. + ## This is required to prevent configure from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `configure` command. + if (path / check).fileExists(): + return + + echo "# Configuring " & path + + if not fileExists(path / "configure"): + for i in @["autogen.sh", "build" / "autogen.sh"]: + if fileExists(path / i): + echo "# Running autogen.sh" + + when defined(unix): + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output + else: + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output + + break + + if not fileExists(path / "configure"): + for i in @["configure.ac", "configure.in"]: + if fileExists(path / i): + echo "# Running autoreconf" + + echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output + + break + + if fileExists(path / "configure"): + echo "# Running configure " & flags + + when defined(unix): + var + cmd = &"cd {path.sanitizePath} && ./configure" + else: + var + cmd = &"cd {path.sanitizePath} && bash ./configure" + if flags.len != 0: + cmd &= &" {flags}" + + echoDebug execAction(cmd).output + + doAssert (path / check).fileExists(), "Configure failed" + +proc getCmakePropertyStr(name, property, value: string): string = + &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" + +proc getCmakeIncludePath*(paths: openArray[string]): string = + ## Create a `cmake` flag to specify custom include paths + ## + ## Result can be included in the `flag` parameter for `cmake()` or + ## the `cmakeFlags` parameter for `getHeader()`. + for path in paths: + result &= path & ";" + result = " -DCMAKE_INCLUDE_PATH=" & result[0 .. ^2].sanitizePath(sep = "/") + +proc setCmakeProperty*(outdir, name, property, value: string) = + ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook + ## for `getHeader()` + ## + ## `set_target_properties(name PROPERTIES property "value")` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + cm.writeFile( + cm.readFile() & getCmakePropertyStr(name, property, value) + ) + +proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = + ## Set a `cmake` property in `outdir / CMakeLists.txt` to specify a custom library output + ## name - usable in the `xxxPreBuild` hook for `getHeader()` + ## + ## `prefix` is typically `lib` + ## `oname` is the library name + ## `suffix` is typically `.a` + ## + ## Sometimes, `cmake` generates non-standard library names - e.g. zlib compiles to + ## `libzlibstatic.a` on Windows. This proc can help rename it to `libzlib.a` so that `getHeader()` + ## can find it after the library is compiled. + ## + ## ``` + ## set_target_properties(name PROPERTIES PREFIX "prefix") + ## set_target_properties(name PROPERTIES OUTPUT_NAME "oname") + ## set_target_properties(name PROPERTIES SUFFIX "suffix") + ## ``` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + var + str = "" + if prefix.len != 0: + str &= getCmakePropertyStr(name, "PREFIX", prefix) + if oname.len != 0: + str &= getCmakePropertyStr(name, "OUTPUT_NAME", oname) + if suffix.len != 0: + str &= getCmakePropertyStr(name, "SUFFIX", suffix) + if str.len != 0: + cm.writeFile(cm.readFile() & str) + +proc setCmakePositionIndependentCode*(outdir: string) = + ## Set a `cmake` directive to create libraries with -fPIC enabled + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + let + pic = "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" + cmd = cm.readFile() + if not cmd.contains(pic): + cm.writeFile( + pic & "\n" & cmd + ) + +proc cmake*(path, check, flags: string) = + ## Run the `cmake` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## `path` will be created since typically `cmake` is run in an + ## empty directory. + ## + ## `check` is a file that will be generated by the `cmake` command. + ## This is required to prevent `cmake` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `cmake` command. + ## Unlike `configure`, it is required since typically it will be the + ## path to the repository, typically `..` when `path` is a subdir. + if (path / check).fileExists(): + return + + echo "# Running cmake " & flags + echo "# Path: " & path + + mkDir(path) + + let + cmd = &"cd {path.sanitizePath} && cmake {flags}" + + echoDebug execAction(cmd).output + + doAssert (path / check).fileExists(), "cmake failed" + +proc make*(path, check: string, flags = "", regex = false) = + ## Run the `make` command to build all binaries in the specified path + ## + ## `check` is a file that will be generated by the `make` command. + ## This is required to prevent `make` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `make` command. + ## + ## `regex` can be set to true if `check` is a regular expression. + ## + ## If `make.exe` is missing and `mingw32-make.exe` is available, it will + ## be copied over to make.exe in the same location. + if findFile(check, path, regex = regex).len != 0: + return + + echo "# Running make " & flags + echo "# Path: " & path + + var + cmd = findExe("make") + + if cmd.len == 0: + cmd = findExe("mingw32-make") + if cmd.len != 0: + cpFile(cmd, cmd.replace("mingw32-make", "make")) + doAssert cmd.len != 0, "Make not found" + + cmd = &"cd {path.sanitizePath} && make" + if flags.len != 0: + cmd &= &" {flags}" + + echoDebug execAction(cmd).output + + doAssert findFile(check, path, regex = regex).len != 0, "make failed" + +proc buildWithCmake(outdir, flags: string): BuildStatus = + if not fileExists(outdir / "Makefile"): + if fileExists(outdir / "CMakeLists.txt"): + if findExe("cmake").len != 0: + var + gen = "" + when defined(Windows): + if findExe("sh").len != 0: + let + uname = execAction("sh -c uname -a").output.toLowerAscii() + if uname.contains("msys"): + gen = "MSYS Makefiles".quoteShell + elif uname.contains("mingw"): + gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" + else: + echo "Unsupported system: " & uname + else: + gen = "MinGW Makefiles".quoteShell + else: + gen = "Unix Makefiles".quoteShell + if findExe("ccache").len != 0: + gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + result.buildPath = outdir / "buildcache" + cmake(result.buildPath, "Makefile", &".. -G {gen} {flags}") + result.built = true + else: + result.error = "cmake capable but cmake executable missing" + else: + result.buildPath = outdir + +proc buildWithAutoConf(outdir, flags: string): BuildStatus = + if not fileExists(outdir / "Makefile"): + if findExe("bash").len != 0: + for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: + if fileExists(outdir / file): + configure(outdir, "Makefile", flags) + result.buildPath = outdir + result.built = true + break + else: + result.error = "configure capable but bash executable missing" + else: + result.buildPath = outdir + +proc flagBuild*(base: string, flags: openArray[string]): string = + ## Simple helper proc to generate flags for `configure`, `cmake`, etc. + ## + ## Every entry in `flags` is replaced into the `base` string and + ## concatenated to the result. + ## + ## E.g. + ## `base = "--disable-$#"` + ## `flags = @["one", "two"]` + ## + ## `flagBuild(base, flags) => " --disable-one --disable-two"` + for i in flags: + result &= " " & base % i diff --git a/nimterop/git.nim b/nimterop/git.nim deleted file mode 100644 index 7a0cf2a..0000000 --- a/nimterop/git.nim +++ /dev/null @@ -1 +0,0 @@ -include build \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 1844a56..495e5d1 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,9 @@ import os, osproc, sets, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, build, globals, getters, grammar, tshelp] +import "."/[build, globals] + +import "."/toastlib/[ast, ast2, getters, grammar, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/ast.nim b/nimterop/toastlib/ast.nim similarity index 99% rename from nimterop/ast.nim rename to nimterop/toastlib/ast.nim index 78501fd..0b62a34 100644 --- a/nimterop/ast.nim +++ b/nimterop/toastlib/ast.nim @@ -2,7 +2,8 @@ import hashes, macros, os, sets, strformat, strutils, tables import regex -import "."/[getters, globals, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[getters, tshelp] proc getHeaderPragma*(gState: State): string = result = diff --git a/nimterop/ast2.nim b/nimterop/toastlib/ast2.nim similarity index 99% rename from nimterop/ast2.nim rename to nimterop/toastlib/ast2.nim index de3b452..ffe961e 100644 --- a/nimterop/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -2,9 +2,9 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/api +import ".."/[globals, treesitter/api] -import "."/[comphelp, exprparser, globals, getters, tshelp] +import "."/[comphelp, exprparser, getters, tshelp] proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined diff --git a/nimterop/comphelp.nim b/nimterop/toastlib/comphelp.nim similarity index 98% rename from nimterop/comphelp.nim rename to nimterop/toastlib/comphelp.nim index e577717..d404c42 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/toastlib/comphelp.nim @@ -2,7 +2,8 @@ import macros, strutils import compiler/[ast, idents, lineinfos, msgs, options, parser, pathutils, renderer] -import "."/[globals, getters, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[getters, tshelp] proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = # Raise exception in parseString() instead of exiting for errors diff --git a/nimterop/exprparser.nim b/nimterop/toastlib/exprparser.nim similarity index 99% rename from nimterop/exprparser.nim rename to nimterop/toastlib/exprparser.nim index 0d2099a..dc9f97e 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -4,9 +4,9 @@ import regex import compiler/[ast, renderer] -import "."/treesitter/[api, c, cpp] - -import "."/[globals, getters, comphelp, tshelp] +import ".."/treesitter/[api, c, cpp] +import ".."/globals +import "."/[getters, comphelp, tshelp] # This version of exprparser should be able to handle: # diff --git a/nimterop/getters.nim b/nimterop/toastlib/getters.nim similarity index 99% rename from nimterop/getters.nim rename to nimterop/toastlib/getters.nim index b092663..b4070f3 100644 --- a/nimterop/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import "."/[build, globals, plugin] +import ".."/[build, globals, plugin] const gReserved = """ addr and as asm diff --git a/nimterop/grammar.nim b/nimterop/toastlib/grammar.nim similarity index 99% rename from nimterop/grammar.nim rename to nimterop/toastlib/grammar.nim index e061a70..52b19e7 100644 --- a/nimterop/grammar.nim +++ b/nimterop/toastlib/grammar.nim @@ -2,7 +2,8 @@ import macros, strformat, strutils, tables import regex -import "."/[ast, getters, globals, lisp, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[ast, getters, lisp, tshelp] type Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] diff --git a/nimterop/lisp.nim b/nimterop/toastlib/lisp.nim similarity index 97% rename from nimterop/lisp.nim rename to nimterop/toastlib/lisp.nim index 8287cf5..57d71a1 100644 --- a/nimterop/lisp.nim +++ b/nimterop/toastlib/lisp.nim @@ -1,4 +1,5 @@ -import "."/[getters, globals] +import ".."/globals +import "."/getters var gTokens: seq[string] diff --git a/nimterop/tshelp.nim b/nimterop/toastlib/tshelp.nim similarity index 99% rename from nimterop/tshelp.nim rename to nimterop/toastlib/tshelp.nim index ad3e89e..91f77ec 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -1,7 +1,8 @@ import sets, strformat, strutils -import "."/[getters, globals] -import "."/treesitter/[api, c, cpp] +import ".."/treesitter/[api, c, cpp] +import ".."/globals +import "."/getters template withCodeAst*(code: string, mode: string, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code From 69e372beeaa4c1aa306efb13237bf274a33a1f05 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 13:52:01 -0500 Subject: [PATCH 188/255] Improve octal fix --- nimterop/build.nim | 5 ++++- nimterop/build/nimconf.nim | 3 ++- nimterop/toastlib/exprparser.nim | 22 ++++++++++------------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 99adeca..0754615 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,7 @@ -import hashes, macros, osproc, sets, strformat, strutils, tables +import hashes, osproc, sets, strformat, strutils + +when not defined(TOAST): + import macros, tables import os except findExe, sleep diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 98fc8fe..3ba5ed0 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,4 +1,4 @@ -import json, macros, os, osproc, sets, strformat, strutils +import json, os, osproc, sets, strformat, strutils when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): @@ -55,6 +55,7 @@ proc getProjectDir*(): string = result = querySetting(projectFull).parentDir() else: # Get from `macros` + import macros result = getProjectPath() else: discard diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index dc9f97e..28b49d3 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -140,15 +140,16 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = var val: BiggestInt flags: TNodeFlags - if number.startsWith("0X") or number.startsWith("0x"): - val = parseHexInt(number) - flags = {nfBase16} - elif number.startsWith("0B") or number.startsWith("0b"): - val = parseBinInt(number) - flags = {nfBase2} - elif number.startsWith("0O") or number.startsWith("0o"): - val = parseOctInt(number) - flags = {nfBase8} + if number.len > 1 and number[0] == '0': + if number[1] in ['x', 'X']: + val = parseHexInt(number) + flags = {nfBase16} + elif number[1] in ['b', 'B']: + val = parseBinInt(number) + flags = {nfBase2} + else: + val = parseOctInt(number) + flags = {nfBase8} else: val = parseInt(number) @@ -201,9 +202,6 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = if number.startsWith("-"): number = number[1 ..< number.len] prefix = "-" - if number.len > 1 and number[0] == '0' and number[1] notin ['x', 'X']: - # Octal 0123 - number = "0o" & number[1 .. ^1] if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): suffix = number[^3 .. ^1] number = number[0 ..< ^3] From c3228683fa91e01c75a8676479ee13a41c7f5a02 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 14:21:59 -0500 Subject: [PATCH 189/255] Fix macros bug --- nimterop/build/nimconf.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 3ba5ed0..c87a60f 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -3,6 +3,8 @@ import json, os, osproc, sets, strformat, strutils when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): import std/compilesettings + else: + import macros else: discard @@ -55,7 +57,6 @@ proc getProjectDir*(): string = result = querySetting(projectFull).parentDir() else: # Get from `macros` - import macros result = getProjectPath() else: discard From d07bd2d71e194a3ed1c0bd5eec9537db26b0bf3f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 22:06:02 -0500 Subject: [PATCH 190/255] findExe sanitizePath --- nimterop/build/shell.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 4f3c316..235f7b5 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -86,7 +86,7 @@ proc findExe*(exe: string): string = (output, ret) = execAction(cmd, die = false) if ret == 0: - return output.splitLines()[0].strip() + return output.splitLines()[0].strip().sanitizePath proc mkDir*(dir: string) = ## Create a directory at compile time From 6c3142a27c186ed4b02bf717ad2dafc37fe81440 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 09:50:00 -0500 Subject: [PATCH 191/255] Fix findFile with regex --- nimterop/build/shell.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 235f7b5..5179798 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -393,7 +393,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s var dir = dir file = file - if not recurse: + # If file = `path/file`, adjust dir = `dir/path` and search for new file + if not (recurse or regex): let pdir = file.parentDir() if pdir.len != 0: From 77380630383f37f08d0ead6438aa4c90966b8d25 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 11:42:50 -0500 Subject: [PATCH 192/255] Fix #231 - const func type, comment in struct --- nimterop/toastlib/ast2.nim | 13 +++--- nimterop/toastlib/tshelp.nim | 2 +- tests/include/tast2.h | 83 ++++++++++++++++++++++++++++++++++-- tests/tast2.nim | 4 ++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index ffe961e..701b0b4 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1238,8 +1238,9 @@ proc addType(gState: State, node: TSNode, union = false) = gState.addTypeObject(node[0], union = union) else: let - fdecl = node[1].anyChildInTree("function_declarator") - adecl = node[1].anyChildInTree("array_declarator") + start = node.getStartAtom() + fdecl = node[start+1].anyChildInTree("function_declarator") + adecl = node[start+1].anyChildInTree("array_declarator") if fdlist.isNil: if adecl.isNil and fdecl.isNil: # typedef X Y; @@ -1341,9 +1342,9 @@ proc addType(gState: State, node: TSNode, union = false) = # First add struct as object decho("addType(): case 6") - gState.addTypeObject(node[0], union = union) + gState.addTypeObject(node[start], union = union) - if node.len > 1 and gState.getNodeVal(node[1]) != "": + if node.len > start+1 and gState.getNodeVal(node[start+1]) != "": # Add any additional names gState.addTypeTyped(node) else: @@ -1357,14 +1358,14 @@ proc addType(gState: State, node: TSNode, union = false) = name = block: var name = "" - for i in 1 ..< node.len: + for i in start+1 ..< node.len: if node[i].getName() == "type_identifier": name = gState.getNodeVal(node[i].getAtom()) name # Now add struct as object with specified name - gState.addTypeObject(node[0], fname = name, istype = true, union = union) + gState.addTypeObject(node[start], fname = name, istype = true, union = union) if name.nBl: # Add any additional names diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 91f77ec..b055259 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -62,7 +62,7 @@ proc getAtom*(node: TSNode): TSNode = if node.getName() in gAtoms: return node elif node.len != 0: - if node[0].getName() == "type_qualifier": + if node[0].getName() in ["type_qualifier", "comment"]: # Skip const, volatile if node.len > 1: return node[1].getAtom() diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 6f569d5..89ad486 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -61,6 +61,7 @@ struct some_struct_s struct parent_struct_s { + /* Random comment */ struct some_struct_s s[SOME_CONST]; }; @@ -108,6 +109,9 @@ typedef struct A20 { char a1; } A20, A21, *A21p; //Expression typedef struct A22 { const int **f1; int *f2[123+132]; } A22; +// #231 +typedef const char *(*A23)(); + //Unions union U1 {int f1; float f2; }; typedef union U2 { const int **f1; int abc[123+132]; } U2; @@ -239,6 +243,7 @@ typedef struct { struct { int f1; } f2; struct NT3 { + /* Random comment */ struct { int f1; union NU1 { @@ -277,6 +282,65 @@ struct TestMyInt { #define C 0x10 #define D "hello" #define E 'c' +#define F 01234 + +#define UEXPR (1234u << 1) +#define ULEXPR (1234ul << 2) +#define ULLEXPR (1234ull << 3) +#define LEXPR (1234l << 4) +#define LLEXPR (1234ll << 5) + +#define SHL1 (1u << 1) +#define SHL2 (1u << 2) +#define SHL3 (1u << 3) +#define COERCE 645635634896ull + 35436 +#define COERCE2 645635634896 + 35436ul +#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define POINTEREXPR (int*)0 +#define POINTERPOINTERPOINTEREXPR (int***)0 +#define BOOL true +#define MATHEXPR (1 + 2/3*20 - 100) +#define ANDEXPR (100 & 11000) +#define CASTEXPR (char) 34 +#define AVAL 100 +#define BVAL 200 +#define EQ1 AVAL <= BVAL +#define EQ2 AVAL >= BVAL +#define EQ3 AVAL > BVAL +#define EQ4 AVAL < BVAL +#define EQ5 AVAL != BVAL +#define EQ6 AVAL == BVAL + +// testing integer out of long int range +#define INT_FAST16_MIN (-9223372036854775807L-1) + +#define SIZEOF sizeof(char) +#define REG_STR "regular string" +#define NOTSUPPORTEDSTR "not a " REG_STR + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" + +#define ALLSHL (SHL1 | SHL2 | SHL3) + +#ifdef NIMTEROP +#define SOME_CONST 8 +#endif + +struct some_struct_s +{ + int x; +}; + +struct parent_struct_s +{ + /* Random comment */ + struct some_struct_s s[SOME_CONST]; +}; + +typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; struct A0; struct A1 {}; @@ -303,10 +367,10 @@ typedef char *(*A11)[3]; typedef struct A0 *A111[12]; typedef int - **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A12)(int, int b, int *c, int *, int /*out*/ *count[4], int (*func)(int, int)), **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); -typedef int A13(int, int, void (*func)(void)); +typedef int (*A13)(int, int, void (*func)(void)); struct A14 { volatile char a1; }; struct A15 { char *a1; const int *a2[1]; }; @@ -320,6 +384,9 @@ typedef struct A20 { char a1; } A20, A21, *A21p; //Expression typedef struct A22 { const int **f1; int *f2[123+132]; } A22; +// #231 +typedef const char *(*A23)(); + //Unions union U1 {int f1; float f2; }; typedef union U2 { const int **f1; int abc[123+132]; } U2; @@ -403,6 +470,16 @@ void int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); +// Issue #174 - type name[] => UncheckedArray[type] +int ucArrFunc1(int text[]); +int ucArrFunc2(int text[][5], int (*func)(int text[])); + +typedef int ucArrType1[][5]; +struct ucArrType2 { + float f1[5][5]; + int *f2[][5]; +}; + typedef struct fieldfuncfunc { int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1, ...))); }; @@ -441,6 +518,7 @@ typedef struct { struct { int f1; } f2; struct NT3 { + /* Random comment */ struct { int f1; union NU1 { @@ -469,7 +547,6 @@ struct TestMyInt { }; - #endif #ifdef __cplusplus diff --git a/tests/tast2.nim b/tests/tast2.nim index 124c4e4..3f626ae 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -333,6 +333,10 @@ checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] +assert A23 is proc(): cstring {.cdecl.} +checkPragmas(A23, pHeaderImp & "cdecl") +var a23: A23 + assert U1 is object assert sizeof(U1) == sizeof(cfloat) checkPragmas(U1, pHeaderBy & @["union"], istype = false) From e44ab30af97b8aceefc2568a83fb59c3e5ac21cd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 12:48:18 -0500 Subject: [PATCH 193/255] Fix rebase issues --- nimterop/build/conan.nim | 2 -- tests/getheader.nims | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 7e1bd8e..3430752 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -387,8 +387,6 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = cleanDir(outdir) - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" - pkg.getConanBuilds() doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" diff --git a/tests/getheader.nims b/tests/getheader.nims index 2fe4d9c..916abcf 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -25,8 +25,8 @@ when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): cmd &= " --gc:arc" testCall(cmd & lrcmd, "No build files found", 1) -testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) -testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) +testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan.io uri", 1) +testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, "Version in both uri", 1) when defined(posix): # stdlib @@ -43,17 +43,15 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) -<<<<<<< HEAD else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) # JBB -testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:libssh2JBB" & sshcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) -testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) -======= ->>>>>>> c35eb74... Add tests for conan, recurse implies preprocess +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0, delete = false) +testCall(cmd & " -d:lzmaJBB -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) @@ -74,3 +72,4 @@ testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & " # conan testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:lzmaConan -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) \ No newline at end of file From a8ea96055d9d1276fff401ffb1866e1f34d6bc46 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 23:20:15 -0500 Subject: [PATCH 194/255] dynlib allow path, don't delete project, fix proc inline comment --- CHANGES.md | 1 + README.md | 2 +- nimterop/build/getheader.nim | 6 ++++++ nimterop/toast.nim | 2 +- nimterop/toastlib/ast2.nim | 5 ++++- nimterop/toastlib/tshelp.nim | 8 +++++++- 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cedf5a5..bf4b50a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. +- The `dynlib` command line parameter to `toast` and `cImport()` can also be the path to a shared library (dll|so|dylib) in place of a Nim const string containing the path. This allows for the traditional use case of passing `"xxxLPath"` to `cImport()` as well as simply passing the path to the library on the command line as is. This allows the creation of standalone cached wrappers as well as the usage of the `--check` and the `--stub` functionality that `toast` provides via `cImport()`. ## Version 0.5.0 diff --git a/README.md b/README.md index ca17b90..af8501a 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Options: -C=, --convention= string "cdecl" calling convention for wrapped procs -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" import symbols from library in specified Nim string + -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path -f=, --feature= Features ast1 flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 3852fbf..69741b0 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -392,6 +392,12 @@ macro getHeader*( else: getLocalPath(header, outdir) + static: + # Don't delete project + when not `nameStd` and (`nameGit` or `nameDL` or `nameConan` or `nameJBB`): + doAssert `outdir`.len != 0, "getHeader():outdir cannot be blank" + doAssert `outdir` != getProjectPath(), "getHeader():outdir cannot be the project path" + const `version`* {.strdefine.} = `verVal` `lname` = diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 495e5d1..e8e14cc 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -240,7 +240,7 @@ when isMainModule: "convention": "calling convention for wrapped procs", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", - "dynlib": "import symbols from library in specified Nim string", + "dynlib": "{.dynlib.} pragma to import symbols - Nim const string or file path", "feature": "flags to enable experimental features", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 701b0b4..0016e29 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1805,7 +1805,10 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # {.pragma: impnameDyn, dynlib: libname.} let dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) - gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) + if '.' in gState.dynlib: + gState.addPragma(root, dynPragma, "dynlib", newStrNode(nkStrLit, gState.dynlib)) + else: + gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) gState.pragmaSection.add dynPragma count += 1 diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index b055259..e7e8d75 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -154,7 +154,13 @@ proc firstChildInTree*(node: TSNode, ntype: string): TSNode = while not cnode.isNil: if cnode.getName() == ntype: return cnode - cnode = cnode[0] + if cnode.len != 0: + for i in 0 ..< cnode.len: + if cnode[i].getName() != "comment": + cnode = cnode[i] + break + else: + cnode = cnode[0] proc anyChildInTree*(node: TSNode, ntype: string): TSNode = # Search for node type anywhere in tree - depth first From d126e9944fee992f46bd51d2f65f3db752249163 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 10:37:38 -0500 Subject: [PATCH 195/255] Remove legacy backend --- CHANGES.md | 2 + README.md | 9 +- nimterop.nimble | 13 +- nimterop/build.nim | 8 +- nimterop/build/getheader.nim | 3 +- nimterop/build/shell.nim | 3 +- nimterop/build/tools.nim | 9 +- nimterop/globals.nim | 27 +- nimterop/toast.nim | 39 +- nimterop/toastlib/ast.nim | 253 ----------- nimterop/toastlib/getters.nim | 125 ------ nimterop/toastlib/grammar.nim | 768 ---------------------------------- nimterop/toastlib/lisp.nim | 60 --- nimterop/toastlib/tshelp.nim | 1 - tests/getheader.nims | 2 +- tests/libssh2.nim | 4 +- tests/rsa.nim | 2 +- 17 files changed, 33 insertions(+), 1295 deletions(-) delete mode 100644 nimterop/toastlib/ast.nim delete mode 100644 nimterop/toastlib/grammar.nim delete mode 100644 nimterop/toastlib/lisp.nim diff --git a/CHANGES.md b/CHANGES.md index bf4b50a..93bb8af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 ### Breaking changes +- The legacy algorithm has been removed as promised. `ast2` is now the default and wrappers no longer need to explicitly specify `-f:ast2` in order to use it. + - All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. diff --git a/README.md b/README.md index af8501a..b054d97 100644 --- a/README.md +++ b/README.md @@ -177,13 +177,13 @@ Now that this is understood, a user might want any combination of the above in t - By default, generated wrappers will include the `{.header, importc.}` pragmas for types and procs. This can be disabled with the `--noHeader | -H` flag to `toast` or `flags = "-H"` param to `cImport()` which will remove `{.header}` for both and `{.importc.}` for types only. - By default, generated wrappers will assume that the user will link the library implementation themselves. The `--dynlib | -l` flag to `toast` or `dynlib = "headerLPath"` param to `cImport()` will configure the wrapper to generate `{.dynlib.}` pragmas for procs. -This results in four cases: +This results in four supported cases: 1. Default: `{.header, importc.}` for both types and procs 2. With `--noHeader`, types will be pure Nim and procs will be just `{.importc.}` 3. With `--dynlib`, types will still be `{.header, importc.}` but procs will be `{.dynlib, importc.}` 4. With `--dynlib` and `--noHeader`, types will be pure Nim, procs will be `{.dynlib, importc.}` -While `ast2` supports all these modes, the legacy backend does not support the third mixed case and will infer `--noHeader` when `--dynlib` is specified (case 4). Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib` for `ast2`. +Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib`. More documentation on on these pragmas can be found in the Nim manual: - [{.importc.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-importc-pragma) @@ -218,7 +218,7 @@ Options: -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path - -f=, --feature= Features ast1 flags to enable experimental features + -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp --nim= string "nim" use a particular Nim executable @@ -226,12 +226,11 @@ Options: -H, --noHeader bool false skip {.header.} pragma in wrapper -o=, --output= string "" file to output content - default: stdout -a, --past bool false print AST output - -g, --pgrammar bool false print grammar --pluginSourcePath= string "" nim file to build and load as a plugin -n, --pnim bool false print Nim output -E=, --prefix= strings {} strip prefix from identifiers -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files, implies --preprocess + -r, --recurse bool false process #include files - implies --preprocess -G=, --replace= strings {} replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers diff --git a/nimterop.nimble b/nimterop.nimble index 1657aed..7be89c6 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.9" +version = "0.6.0" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -55,15 +55,12 @@ task test, "Test": execTest "tests/tast2.nim", "-d:NOHEADER" execTest "tests/tnimterop_c.nim" - execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-H\"" execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast tests/toast.cfg tests/include/toast.h" - execCmd "./nimterop/toast tests/toast.cfg -f:ast2 tests/include/toast.h" execTest "tests/tpcre.nim" - execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" when defined(Linux): execTest "tests/rsa.nim" @@ -72,12 +69,10 @@ task test, "Test": # Platform specific tests when defined(Windows): execTest "tests/tmath.nim" - execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tmath.nim", "-d:FLAGS=\"-H\"" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): execTest "tests/tsoloud.nim" - execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tsoloud.nim", "-d:FLAGS=\"-H\"" # getHeader tests withDir("tests"): diff --git a/nimterop/build.nim b/nimterop/build.nim index 0754615..672cb58 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -14,12 +14,6 @@ var gNimExe* = "" # Misc helpers -proc echoDebug(str: string) = - let str = "\n# " & str.strip().replace("\n", "\n# ") - when nimvm: - if gDebugCT: echo str - else: - if gDebug: echo str proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) @@ -34,7 +28,7 @@ proc getCurrentNimCompiler*(): string = else: result = gNimExe -template fixOutDir() {.dirty.} = +template fixOutDir() {.dirty, used.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 69741b0..216b599 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -178,7 +178,6 @@ proc getLocalPath(header, outdir: string): string = proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = var lpath = findFile(lname, outdir, regex = true) - makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" makePath = outdir if lpath.len != 0: @@ -200,7 +199,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): - make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) + make(buildStatus.buildPath, lname, makeFlags, regex = true) buildStatus.built = true let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 5179798..bc07285 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -455,7 +455,8 @@ proc linkLibs*(names: openArray[string], staticLink = true): string = for res in resSet: result &= " " & res -proc getNumProcs(): string = +proc getNumProcs*(): string = + ## Get number of processors when defined(Windows): getEnv("NUMBER_OF_PROCESSORS").strip() elif defined(linux): diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index 772c48b..c6a4283 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -9,6 +9,13 @@ type buildPath: string error: string +proc echoDebug(str: string) = + let str = "\n# " & str.strip().replace("\n", "\n# ") + when nimvm: + if gDebugCT: echo str + else: + if gDebug: echo str + proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -193,7 +200,7 @@ proc make*(path, check: string, flags = "", regex = false) = cpFile(cmd, cmd.replace("mingw32-make", "make")) doAssert cmd.len != 0, "Make not found" - cmd = &"cd {path.sanitizePath} && make" + cmd = &"cd {path.sanitizePath} && make -j {getNumProcs()}" if flags.len != 0: cmd &= &" {flags}" diff --git a/nimterop/globals.nim b/nimterop/globals.nim index c79f4a5..fa8f8de 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -3,8 +3,6 @@ import tables when defined(TOAST): import sets, sequtils, strutils - import regex - import "."/plugin import compiler/[ast, idents, modulegraphs, options] @@ -13,7 +11,7 @@ when defined(TOAST): type Feature* = enum - ast1, ast2 + ast2 State* = ref object # Command line arguments to toast - some forwarded from cimport.nim @@ -72,12 +70,6 @@ type # Controls whether or not the current expression # should validate idents against currently defined idents skipIdentValidation*: bool - - # Legacy AST fields, remove when ast2 becomes default - constStr*, enumStr*, procStr*, typeStr*: string - commentStr*, debugStr*, skipStr*: string - data*: seq[tuple[name, val: string]] - nodeBranch*: seq[string] else: # cimport.nim specific compile*: seq[string] # `cCompile()` list of files already processed @@ -114,23 +106,6 @@ when defined(TOAST): ].concat(toSeq(gExpressions.items)) type - Kind* = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast* = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - tonim*: proc (ast: ref Ast, node: TSNode, gState: State) - regex*: Regex - - AstTable* {.used.} = TableRef[string, seq[ref Ast]] - Status* = enum success, unknown, error diff --git a/nimterop/toast.nim b/nimterop/toast.nim index e8e14cc..b7a993a 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,9 +4,9 @@ import "."/treesitter/[api, c, cpp] import "."/[build, globals] -import "."/toastlib/[ast, ast2, getters, grammar, tshelp] +import "."/toastlib/[ast2, getters, tshelp] -proc process(gState: State, path: string, astTable: AstTable) = +proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" if gState.mode.Bl: @@ -21,10 +21,7 @@ proc process(gState: State, path: string, astTable: AstTable) = if gState.past: gecho gState.printLisp(root) elif gState.pnim: - if Feature.ast2 in gState.feature: - ast2.parseNim(gState, path, root) - elif Feature.ast1 in gState.feature: - ast.parseNim(gState, path, root, astTable) + parseNim(gState, path, root) elif gState.preprocess: gecho gState.code @@ -43,7 +40,6 @@ proc main( noHeader = false, output = "", past = false, - pgrammar = false, pluginSourcePath: string = "", pnim = false, prefix: seq[string] = @[], @@ -84,10 +80,6 @@ proc main( build.gDebug = gState.debug build.gNimExe = gState.nim - # Default `ast` mode - if gState.feature.Bl: - gState.feature.add Feature.ast1 - # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() @@ -137,31 +129,14 @@ proc main( if gState.debug: echo &"# Writing output to {outputFile}\n" - # Process grammar into AST - let - astTable = - if Feature.ast1 in gState.feature: - parseGrammar() - else: - nil - - if pgrammar: - if Feature.ast1 in gState.feature: - # Print AST of grammar - gState.printGrammar(astTable) - elif source.nBl: + if source.nBl: # Print source after preprocess or Nim output if gState.pnim: gState.initNim() for src in source: - gState.process(src.expandSymlinkAbs(), astTable) + gState.process(src.expandSymlinkAbs()) if gState.pnim: - if Feature.ast2 in gState.feature: - ast2.printNim(gState) - elif Feature.ast1 in gState.feature: - ast.printNim(gState) - gecho """{.hint: "The legacy wrapper generation algorithm is deprecated and will be removed in the next release of Nimterop.".}""" - gecho """{.hint: "Refer to CHANGES.md for details on migrating to the new backend.".}""" + printNim(gState) # Close outputFile if outputFile.len != 0: @@ -249,7 +224,6 @@ when isMainModule: "noHeader": "skip {.header.} pragma in wrapper", "output": "file to output content - default: stdout", "past": "print AST output", - "pgrammar": "print grammar", "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", "prefix": "strip prefix from identifiers", @@ -273,7 +247,6 @@ when isMainModule: "noHeader": 'H', "output": 'o', "past": 'a', - "pgrammar": 'g', "pnim": 'n', "prefix": 'E', "preprocess": 'p', diff --git a/nimterop/toastlib/ast.nim b/nimterop/toastlib/ast.nim deleted file mode 100644 index 0b62a34..0000000 --- a/nimterop/toastlib/ast.nim +++ /dev/null @@ -1,253 +0,0 @@ -import hashes, macros, os, sets, strformat, strutils, tables - -import regex - -import ".."/[globals, treesitter/api] -import "."/[getters, tshelp] - -proc getHeaderPragma*(gState: State): string = - result = - if not gState.noHeader and gState.dynlib.Bl: - &", header: {gState.currentHeader}" - else: - "" - -proc getDynlib*(gState: State): string = - result = - if gState.dynlib.nBl: - &", dynlib: {gState.dynlib}" - else: - "" - -proc getImportC*(gState: State, origName, nimName: string): string = - if nimName != origName: - result = &"importc: \"{origName}\"{gState.getHeaderPragma()}" - else: - result = gState.impShort - -proc getPragma*(gState: State, pragmas: varargs[string]): string = - result = "" - for pragma in pragmas.items(): - if pragma.nBl: - result &= pragma & ", " - if result.nBl: - result = " {." & result[0 .. ^3] & ".}" - - result = result.replace(gState.impShort & ", cdecl", gState.impShort & "C") - - let - dy = gState.getDynlib() - - if ", cdecl" in result and dy.nBl: - result = result.replace(".}", dy & ".}") - -proc saveNodeData(node: TSNode, gState: State): bool = - let name = $node.tsNodeType() - - # Atoms are nodes whose values are to be saved - if name in gAtoms: - let - pname = node.getPxName(1) - ppname = node.getPxName(2) - pppname = node.getPxName(3) - ppppname = node.getPxName(4) - - var - val = gState.getNodeVal(node) - - # Skip since value already obtained from parent atom - if name == "primitive_type" and pname == "sized_type_specifier": - return true - - # Skip since value already obtained from parent expression - if name in ["number_literal", "identifier"] and pname in gExpressions: - return true - - # Add reference point in saved data for bitfield_clause - if name in ["number_literal"] and pname == "bitfield_clause": - gState.data.add(("bitfield_clause", val)) - return true - - # Process value as a type - if name in ["primitive_type", "sized_type_specifier"]: - val = val.getType() - - if node.tsNodePrevNamedSibling().tsNodeIsNull(): - if pname == "pointer_declarator": - if ppname notin ["function_declarator", "array_declarator"]: - gState.data.add(("pointer_declarator", "")) - elif ppname == "array_declarator": - gState.data.add(("array_pointer_declarator", "")) - - # Double pointer - if ppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - elif pname in ["function_declarator", "array_declarator"]: - if ppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - if pppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - - gState.data.add((name, val)) - - if pname == "pointer_declarator" and - ppname == "function_declarator": - if name == "field_identifier": - if pppname == "pointer_declarator": - gState.data.insert(("pointer_declarator", ""), gState.data.len-1) - if ppppname == "pointer_declarator": - gState.data.insert(("pointer_declarator", ""), gState.data.len-1) - gState.data.add(("function_declarator", "")) - elif name == "identifier": - gState.data.add(("pointer_declarator", "")) - - # Save node value for a top-level expression - elif name in gExpressions and name != "escape_sequence": - if $node.tsNodeParent.tsNodeType() notin gExpressions: - gState.data.add((name, gState.getNodeVal(node))) - - elif name in ["abstract_pointer_declarator", "enumerator", "field_declaration", "function_declarator"]: - gState.data.add((name.replace("abstract_", ""), "")) - - return true - -proc searchAstForNode(ast: ref Ast, node: TSNode, gState: State): bool = - let - childNames = node.getTSNodeNamedChildNames().join() - - if ast.isNil: - return - - if gState.debug: - gState.nodeBranch.add $node.tsNodeType() - gecho "#" & spaces(gState.nodeBranch.len * 2) & gState.nodeBranch[^1] - - if ast.children.nBl: - if childNames.contains(ast.regex) or - (childNames.Bl and ast.recursive): - if node.getTSNodeNamedChildCountSansComments() != 0: - var flag = true - - for i in 0 .. node.tsNodeNamedChildCount()-1: - if $node.tsNodeNamedChild(i).tsNodeType() != "comment": - let - nodeChild = node.tsNodeNamedChild(i) - astChild = - if not ast.recursive: - ast.getAstChildByName($nodeChild.tsNodeType()) - else: - ast - - if not searchAstForNode(astChild, nodeChild, gState): - flag = false - break - - if flag: - result = node.saveNodeData(gState) - else: - result = node.saveNodeData(gState) - else: - if gState.debug: - gecho "#" & spaces(gState.nodeBranch.len * 2) & &" {ast.getRegexForAstChildren()} !=~ {childNames}" - elif node.getTSNodeNamedChildCountSansComments() == 0: - result = node.saveNodeData(gState) - - if gState.debug: - discard gState.nodeBranch.pop() - if gState.nodeBranch.Bl: - gecho "" - -proc searchAst(root: TSNode, astTable: AstTable, gState: State) = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.tsNodeIsNull() and depth > -1: - let - name = $node.tsNodeType() - if name in astTable: - for ast in astTable[name]: - if gState.debug: - gecho "\n# " & gState.getNodeVal(node).replace("\n", "\n# ") & "\n" - if searchAstForNode(ast, node, gState): - ast.tonim(ast, node, gState) - if gState.debug: - gState.debugStr &= "\n# " & gState.data.join("\n# ") & "\n" - break - gState.data = @[] - else: - break - - if $node.tsNodeType() notin astTable and node.tsNodeNamedChildCount() != 0: - nextnode = node.tsNodeNamedChild(0) - depth += 1 - else: - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.tsNodeIsNull(): - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - if node == root: - break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break - -proc parseNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = - # Generate Nim from tree-sitter AST root node - var - fp = fullpath.replace("\\", "/") - - gState.currentHeader = getCurrentHeader(fullpath) - gState.impShort = gState.currentHeader.replace("header", "imp") - gState.sourceFile = fullpath - - if not gState.noHeader and gState.dynlib.Bl: - gState.constStr &= &"\n {gState.currentHeader} {{.used.}} = \"{fp}\"" - - root.searchAst(astTable, gState) - -proc printNim*(gState: State) = - # Print Nim generated by parseNim() - if gState.enumStr.nBl: - gecho &"{gState.enumStr}\n" - - gState.constStr = gState.getOverrideFinal(nskConst) & gState.constStr - if gState.constStr.nBl: - gecho &"const{gState.constStr}\n" - - gecho &""" -{{.pragma: {gState.impShort}, importc{gState.getHeaderPragma()}.}} -{{.pragma: {gState.impShort}C, {gState.impShort}, cdecl{gState.getDynlib()}.}} -""" - - gState.typeStr = gState.getOverrideFinal(nskType) & gState.typeStr - if gState.typeStr.nBl: - gecho &"type{gState.typeStr}\n" - - gState.procStr = gState.getOverrideFinal(nskProc) & gState.procStr - if gState.procStr.nBl: - gecho &"{gState.procStr}\n" - - gecho "{.pop.}" - - if gState.debug: - if gState.debugStr.nBl: - gecho gState.debugStr - - if gState.skipStr.nBl: - let - hash = gState.skipStr.hash().abs() - sname = getTempDir() / &"nimterop_{$hash}.h" - gecho &"# Writing skipped definitions to {sname}\n" - writeFile(sname, gState.skipStr) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index b4070f3..0c1b784 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -304,131 +304,6 @@ proc getPreprocessor*(gState: State, fullpath: string) = rdata.add line gState.code = rdata.join("\n") -converter toString*(kind: Kind): string = - return case kind: - of exactlyOne: - "" - of oneOrMore: - "+" - of zeroOrMore: - "*" - of zeroOrOne: - "?" - of orWithNext: - "!" - -converter toKind*(kind: string): Kind = - return case kind: - of "+": - oneOrMore - of "*": - zeroOrMore - of "?": - zeroOrOne - of "!": - orWithNext - else: - exactlyOne - -proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool] = - if name[0] == '^': - result.recursive = true - result.name = name[1 .. ^1] - else: - result.name = name - result.kind = $name[^1] - - if result.kind != exactlyOne: - result.name = result.name[0 .. ^2] - -proc getRegexForAstChildren*(ast: ref Ast): string = - result = "^" - for i in 0 .. ast.children.len-1: - let - kind: string = ast.children[i].kind - begin = if result[^1] == '|': "" else: "(?:" - case kind: - of "!": - result &= &"{begin}{ast.children[i].name}|" - else: - result &= &"{begin}{ast.children[i].name}){kind}" - result &= "$" - -proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = - for i in 0 .. ast.children.len-1: - if name in ast.children[i].name.split("|"): - return ast.children[i] - - if ast.children.len == 1 and ast.children[0].name == ".": - return ast.children[0] - -proc getNimExpression*(gState: State, expr: string, name = ""): string = - # Convert C/C++ expression into Nim - cast identifiers to `name` if specified - var - clean = expr.multiReplace([("\n", " "), ("\r", "")]) - ident = "" - gen = "" - hex = false - - for i in 0 .. clean.len: - if i != clean.len: - if clean[i] in IdentChars: - if clean[i] in Digits and ident.Bl: - # Identifiers cannot start with digits - gen = $clean[i] - elif clean[i] in HexDigits and hex == true: - # Part of a hex number - gen = $clean[i] - elif i > 0 and i < clean.len-1 and clean[i] in ['x', 'X'] and - clean[i-1] == '0' and clean[i+1] in HexDigits: - # Found a hex number - gen = $clean[i] - hex = true - else: - # Part of an identifier - ident &= clean[i] - hex = false - else: - gen = (block: - if (i == 0 or clean[i-1] != '\'') or - (i == clean.len - 1 or clean[i+1] != '\''): - # If unquoted, convert logical ops to Nim - case clean[i] - of '^': " xor " - of '&': " and " - of '|': " or " - of '~': " not " - else: $clean[i] - else: - $clean[i] - ) - hex = false - - if i == clean.len or gen.nBl: - # Process identifier - if ident.nBl: - # Issue #178 - if ident != "_": - ident = gState.getIdentifier(ident, nskConst, name) - if name.nBl and ident in gState.constIdentifiers: - ident = ident & "." & name - result &= ident - ident = "" - result &= gen - gen = "" - - # Convert shift ops to Nim - result = result.multiReplace([ - ("<<", " shl "), (">>", " shr ") - ]) - -proc getComments*(gState: State, strip = false): string = - if not gState.noComments and gState.commentStr.nBl: - result = "\n" & gState.commentStr - if strip: - result = result.replace("\n ", "\n") - gState.commentStr = "" - # Plugin related proc dll*(path: string): string = diff --git a/nimterop/toastlib/grammar.nim b/nimterop/toastlib/grammar.nim deleted file mode 100644 index 52b19e7..0000000 --- a/nimterop/toastlib/grammar.nim +++ /dev/null @@ -1,768 +0,0 @@ -import macros, strformat, strutils, tables - -import regex - -import ".."/[globals, treesitter/api] -import "."/[ast, getters, lisp, tshelp] - -type - Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] - -proc getPtrType(str: string): string = - result = case str: - of "ptr cchar": - "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": - "pointer" - of "ptr ptr object": - "ptr pointer" - of "ptr FILE": - "File" - else: - str - -proc getLit(str: string): string = - # Used to convert #define literals into const - let - str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - - if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal - str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal - str.contains(re"^'[[:ascii:]]'$") or # char - str.contains(re"""^"[[:ascii:]]+"$"""): # char * - return str - -proc initGrammar(): Grammar = - # #define X Y - result.add((""" - (preproc_def - (identifier) - (preproc_arg) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# define X Y" - - let - name = gState.data[0].val - nname = gState.getIdentifier(name, nskConst) - val = gState.data[1].val.getLit() - - if not nname.nBl: - let - override = gState.getOverride(name, nskConst) - if override.nBl: - gState.constStr &= &"{gState.getComments()}\n{override}" - else: - gState.constStr &= &"{gState.getComments()}\n # Const '{name}' skipped" - if gState.debug: - gState.skipStr &= &"\n{gState.getNodeVal(node)}" - elif val.nBl and gState.addNewIdentifer(nname): - gState.constStr &= &"{gState.getComments()}\n {nname}* = {val}" - )) - - let - typeGrammar = """ - (type_qualifier?) - (primitive_type|type_identifier?) - (type_qualifier?) - (sized_type_specifier? - (primitive_type?) - ) - (struct_specifier|union_specifier|enum_specifier? - (type_identifier) - ) - """ - - arrGrammar = &""" - (array_declarator! - (pointer_declarator! - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - (type_identifier|identifier) - (identifier|number_literal) - ) - """ - - paramListGrammar = &""" - (parameter_list - (parameter_declaration* - {typeGrammar} - (identifier|type_identifier?) - (pointer_declarator? - (type_qualifier?) - (pointer_declarator! - (type_qualifier?) - {arrGrammar} - (identifier|type_identifier) - ) - {arrGrammar} - (identifier|type_identifier) - ) - {arrGrammar} - (abstract_pointer_declarator? - (abstract_pointer_declarator?) - ) - ) - ) - """ - - funcGrammar = &""" - (function_declarator* - (identifier|type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier|identifier) - ) - {paramListGrammar} - (noexcept|throw_specifier?) - ) - """ - - template funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen: untyped): untyped = - ptyp = gState.getIdentifier(gState.data[i].val, nskType, fname).getType() - - pptr = "" - while i+1 < gState.data.len and gState.data[i+1].name == "pointer_declarator": - pptr &= "ptr " - i += 1 - - if i+1 < gState.data.len and gState.data[i+1].name == "identifier": - pname = gState.getIdentifier(gState.data[i+1].val, nskParam, fname) - i += 2 - else: - pname = "a" & $count - count += 1 - i += 1 - - if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: - flen = gState.data[i].val - if gState.data[i].name == "identifier": - flen = gState.getIdentifier(flen, nskConst, fname) - - pout &= &"{pname}: array[{flen}, {getPtrType(pptr&ptyp)}], " - i += 1 - elif pptr.nBl or ptyp != "object": - pout &= &"{pname}: {getPtrType(pptr&ptyp)}, " - - # typedef int X - # typedef X Y - # typedef struct X Y - # typedef ?* Y - result.add((&""" - (type_definition - {typeGrammar} - (type_identifier!) - {arrGrammar} - (pointer_declarator! - (pointer_declarator! - (type_identifier!) - {arrGrammar} - {funcGrammar} - ) - (type_identifier!) - {arrGrammar} - {funcGrammar} - ) - {funcGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef X Y" - - var - i = 0 - typ = gState.getIdentifier(gState.data[i].val, nskType, "IgnoreSkipSymbol").getType() - name = "" - nname = "" - tptr = "" - aptr = "" - pragmas: seq[string] = @[] - - i += 1 - while i < gState.data.len and "pointer" in gState.data[i].name: - case gState.data[i].name: - of "pointer_declarator": - tptr &= "ptr " - i += 1 - of "array_pointer_declarator": - aptr &= "ptr " - i += 1 - - if i < gState.data.len: - name = gState.data[i].val - nname = gState.getIdentifier(name, nskType) - i += 1 - - if not gState.noHeader and gState.dynlib.Bl: - pragmas.add gState.getImportC(name, nname) - - let - pragma = gState.getPragma(pragmas) - - if not nname.nBl: - let - override = gState.getOverride(name, nskType) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif nname notin gTypeMap and typ.nBl and gState.addNewIdentifer(nname): - if i < gState.data.len and gState.data[^1].name == "function_declarator": - var - fname = nname - pout, pname, ptyp, pptr = "" - count = 1 - flen = "" - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - break - - funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - - if tptr.nBl or typ != "object": - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}" - else: - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}) {{.cdecl.}}" - else: - if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: - var - flen = gState.data[i].val - if gState.data[i].name == "identifier": - flen = gState.getIdentifier(flen, nskConst, nname) - - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]" - else: - if nname == typ: - pragmas.add "incompleteStruct" - let - pragma = gState.getPragma(pragmas) - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" - else: - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {getPtrType(tptr&typ)}" - )) - - proc pDupTypeCommon(nname: string, fend: int, gState: State, isEnum=false) = - if gState.debug: - gState.debugStr &= "\n# pDupTypeCommon()" - - var - dname = gState.data[^1].val - ndname = gState.getIdentifier(dname, nskType) - dptr = - if fend == 2: - "ptr " - else: - "" - - if ndname.nBl and ndname != nname: - if isEnum: - if gState.addNewIdentifer(ndname): - gState.enumStr &= &"{gState.getComments(true)}\ntype {ndname}* = {dptr}{nname}" - else: - if gState.addNewIdentifer(ndname): - let - pragma = gState.getPragma(gState.getImportc(dname, ndname), "bycopy") - gState.typeStr &= - &"{gState.getComments()}\n {ndname}*{pragma} = {dptr}{nname}" - - proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = - if gState.debug: - gState.debugStr &= "\n# pStructCommon" - - var - nname = gState.getIdentifier(name, nskType) - prefix = "" - union = "" - - case $node.tsNodeType(): - of "struct_specifier": - prefix = "struct " - of "union_specifier": - prefix = "union " - union = ", union" - of "type_definition": - if node.getTSNodeNamedChildCountSansComments() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - let - nchild = $node.tsNodeNamedChild(i).tsNodeType() - if nchild != "comment": - case nchild: - of "struct_specifier": - if fstart == 1: - prefix = "struct " - of "union_specifier": - if fstart == 1: - prefix = "union " - union = ", union" - break - - if not nname.nBl: - let - override = gState.getOverride(name, nskType) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif gState.addNewIdentifer(nname): - if gState.data.len == 1: - gState.typeStr &= &"{gState.getComments()}\n {nname}* {{.bycopy{union}.}} = object" - else: - var - pragmas: seq[string] = @[] - if not gState.noHeader and gState.dynlib.Bl: - pragmas.add gState.getImportC(prefix & name, nname) - pragmas.add "bycopy" - if union.nBl: - pragmas.add "union" - - let - pragma = gState.getPragma(pragmas) - - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" - - var - i = fstart - ftyp, fname: string - fptr = "" - aptr = "" - flen = "" - while i < gState.data.len-fend: - fptr = "" - aptr = "" - if gState.data[i].name == "field_declaration": - i += 1 - continue - - if gState.data[i].name notin ["field_identifier", "pointer_declarator", "array_pointer_declarator"]: - ftyp = gState.getIdentifier(gState.data[i].val, nskType, nname).getType() - i += 1 - - while i < gState.data.len-fend and "pointer" in gState.data[i].name: - case gState.data[i].name: - of "pointer_declarator": - fptr &= "ptr " - i += 1 - of "array_pointer_declarator": - aptr &= "ptr " - i += 1 - - fname = gState.getIdentifier(gState.data[i].val, nskField, nname) - - if i+1 < gState.data.len-fend and gState.data[i+1].name in gEnumVals: - # Struct field is an array where size is an expression - var - flen = gState.getNimExpression(gState.data[i+1].val) - if "/" in flen: - flen = &"({flen}).int" - gState.typeStr &= &"{gState.getComments()}\n {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]" - i += 2 - elif i+1 < gState.data.len-fend and gState.data[i+1].name == "bitfield_clause": - let - size = gState.data[i+1].val - gState.typeStr &= &"{gState.getComments()}\n {fname}* {{.bitsize: {size}.}} : {getPtrType(fptr&ftyp)} " - i += 2 - elif i+1 < gState.data.len-fend and gState.data[i+1].name == "function_declarator": - var - pout, pname, ptyp, pptr = "" - count = 1 - - i += 2 - while i < gState.data.len-fend: - if gState.data[i].name == "function_declarator": - i += 1 - continue - - if gState.data[i].name == "field_declaration": - break - - funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - if fptr.nBl or ftyp != "object": - gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}" - else: - gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}) {{.cdecl.}}" - i += 1 - else: - if ftyp == "object": - gState.typeStr &= &"{gState.getComments()}\n {fname}*: pointer" - else: - gState.typeStr &= &"{gState.getComments()}\n {fname}*: {getPtrType(fptr&ftyp)}" - i += 1 - - if node.tsNodeType() == "type_definition" and - gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, gState, false) - - let - fieldGrammar = &""" - (field_identifier!) - (bitfield_clause! - (number_literal) - ) - (array_declarator! - (field_identifier!) - (pointer_declarator - (pointer_declarator! - (field_identifier) - ) - (field_identifier) - ) - (^$1+) - ) - (function_declarator+ - (pointer_declarator - (pointer_declarator! - (field_identifier) - ) - (field_identifier) - ) - {paramListGrammar} - ) - """ % gEnumVals.join("|") - - fieldListGrammar = &""" - (field_declaration_list? - (field_declaration+ - {typeGrammar} - (pointer_declarator! - (pointer_declarator! - {fieldGrammar} - ) - {fieldGrammar} - ) - {fieldGrammar} - ) - ) - """ - - # struct X {} - result.add((&""" - (struct_specifier|union_specifier - (type_identifier) - {fieldListGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# struct X {}" - - pStructCommon(ast, node, gState.data[0].val, 1, 1, gState) - )) - - # typedef struct X {} - result.add((&""" - (type_definition - (struct_specifier|union_specifier - (type_identifier?) - {fieldListGrammar} - ) - (type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef struct X {}" - - var - fstart = 0 - fend = 1 - - if gState.data[^2].name == "pointer_declarator": - fend = 2 - - if gState.data.len > 1 and - gState.data[0].name == "type_identifier" and - gState.data[1].name notin ["field_identifier", "pointer_declarator"]: - - fstart = 1 - pStructCommon(ast, node, gState.data[0].val, fstart, fend, gState) - else: - pStructCommon(ast, node, gState.data[^1].val, fstart, fend, gState) - )) - - proc pEnumCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = - if gState.debug: - gState.debugStr &= "\n# pEnumCommon()" - - let nname = - if name.Bl: - getUniqueIdentifier(gState, "Enum") - else: - gState.getIdentifier(name, nskType) - - if nname.nBl and gState.addNewIdentifer(nname): - gState.enumStr &= &"{gState.getComments(true)}\ndefineEnum({nname})" - - var - i = fstart - count = 0 - while i < gState.data.len-fend: - if gState.data[i].name == "enumerator": - i += 1 - continue - - let - fname = gState.getIdentifier(gState.data[i].val, nskEnumField) - - if i+1 < gState.data.len-fend and - gState.data[i+1].name in gEnumVals: - if fname.nBl and gState.addNewIdentifer(fname): - gState.constStr &= &"{gState.getComments()}\n {fname}* = ({gState.getNimExpression(gState.data[i+1].val)}).{nname}" - try: - count = gState.data[i+1].val.parseInt() + 1 - except: - count += 1 - i += 2 - else: - if fname.nBl and gState.addNewIdentifer(fname): - gState.constStr &= &"{gState.getComments()}\n {fname}* = {count}.{nname}" - i += 1 - count += 1 - - if node.tsNodeType() == "type_definition" and - gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, gState, true) - - # enum X {} - result.add((""" - (enum_specifier - (type_identifier?) - (enumerator_list - (enumerator+ - (identifier?) - (^$1+) - ) - ) - ) - """ % gEnumVals.join("|"), - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# enum X {}" - - var - name = "" - offset = 0 - - if gState.data[0].name == "type_identifier": - name = gState.data[0].val - offset = 1 - - pEnumCommon(ast, node, name, offset, 0, gState) - )) - - # typedef enum {} X - result.add((&""" - (type_definition - {result[^1].grammar} - (type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef enum {}" - - var - fstart = 0 - fend = 1 - - if gState.data[^2].name == "pointer_declarator": - fend = 2 - - if gState.data[0].name == "type_identifier": - fstart = 1 - - pEnumCommon(ast, node, gState.data[0].val, fstart, fend, gState) - else: - pEnumCommon(ast, node, gState.data[^1].val, fstart, fend, gState) - )) - - # typ function(typ param1, ...) - result.add((&""" - (declaration - (storage_class_specifier?) - {typeGrammar} - (pointer_declarator! - (pointer_declarator! - {funcGrammar} - ) - {funcGrammar} - ) - {funcGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typ function" - - var - fptr = "" - i = 1 - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - i += 1 - continue - - fptr = "" - while i < gState.data.len and gState.data[i].name == "pointer_declarator": - fptr &= "ptr " - i += 1 - - var - fname = gState.data[i].val - fnname = gState.getIdentifier(fname, nskProc) - pout, pname, ptyp, pptr = "" - count = 1 - flen = "" - fVar = false - - i += 1 - if i < gState.data.len and gState.data[i].name == "pointer_declarator": - fVar = true - i += 1 - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - break - - funcParamCommon(fnname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - - if not fnname.nBl: - let - override = gState.getOverride(fname, nskProc) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif gState.addNewIdentifer(fnname): - let - ftyp = gState.getIdentifier(gState.data[0].val, nskType, fnname).getType() - pragma = gState.getPragma(gState.getImportC(fname, fnname), "cdecl") - - if fptr.nBl or ftyp != "object": - if fVar: - gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}): {getPtrType(fptr&ftyp)}{{.cdecl.}}" - else: - gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}" - else: - if fVar: - gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}){{.cdecl.}}" - else: - gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}){pragma}" - )) - - # // comment - result.add((&""" - (comment - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - let - cmt = $gState.getNodeVal(node) - - for line in cmt.splitLines(): - let - line = line.multiReplace([("//", ""), ("/*", ""), ("*/", "")]) - - gState.commentStr &= &"\n # {line.strip(leading=false)}" - )) - - # // unknown - result.add((&""" - (type_definition|struct_specifier|union_specifier|enum_specifier|declaration - (^.*) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - var - done = false - for i in gState.data: - case $node.tsNodeType() - of "declaration": - if i.name == "identifier": - let - override = gState.getOverride(i.val, nskProc) - - if override.nBl: - gState.procStr &= &"{gState.getComments(true)}\n{override}" - done = true - break - else: - gState.procStr &= &"{gState.getComments(true)}\n# Declaration '{i.val}' skipped" - - else: - if i.name == "type_identifier": - let - override = gState.getOverride(i.val, nskType) - - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - done = true - break - else: - gState.typeStr &= &"{gState.getComments()}\n # Type '{i.val}' skipped" - - if gState.debug and not done: - gState.skipStr &= &"\n{gState.getNodeVal(node)}" - )) - -proc initRegex(ast: ref Ast) = - if ast.children.nBl: - if not ast.recursive: - for child in ast.children: - child.initRegex() - - var - reg: string - try: - reg = ast.getRegexForAstChildren() - ast.regex = reg.re() - except: - echo reg - raise newException(Exception, getCurrentExceptionMsg()) - -proc parseGrammar*(): AstTable = - const grammars = initGrammar() - - result = newTable[string, seq[ref Ast]]() - for i in 0 .. grammars.len-1: - var - ast = grammars[i].grammar.parseLisp() - - ast.tonim = grammars[i].call - ast.initRegex() - for n in ast.name.split("|"): - if n notin result: - result[n] = @[ast] - else: - result[n].add(ast) - -proc printGrammar*(gState: State, astTable: AstTable) = - for name in astTable.keys(): - for ast in astTable[name]: - gecho ast.printAst() diff --git a/nimterop/toastlib/lisp.nim b/nimterop/toastlib/lisp.nim deleted file mode 100644 index 57d71a1..0000000 --- a/nimterop/toastlib/lisp.nim +++ /dev/null @@ -1,60 +0,0 @@ -import ".."/globals -import "."/getters - -var - gTokens: seq[string] - idx = 0 - -proc tokenize(tree: string) = - var collect = "" - - gTokens = @[] - idx = 0 - for i in tree: - case i: - of ' ', '\n', '\r', '(', ')': - if collect.nBl: - gTokens.add(collect) - collect = "" - if i in ['(', ')']: - gTokens.add($i) - else: - collect &= $i - -proc readFromTokens(): ref Ast = - if idx == gTokens.len: - doAssert false, "Bad AST " & $(idx: idx) - - if gTokens[idx] == "(": - if gTokens.len - idx < 2: - doAssert false, "Corrupt AST " & $(gTokensLen: gTokens.len, idx: idx) - result = new(Ast) - (result.name, result.kind, result.recursive) = gTokens[idx+1].getNameKind() - result.children = @[] - if result.recursive: - result.children.add(result) - idx += 2 - while gTokens[idx] != ")": - var res = readFromTokens() - if not res.isNil: - result.children.add(res) - elif gTokens[idx] == ")": - doAssert false, "Poor AST " & $(idx: idx) - - idx += 1 - -proc printAst*(node: ref Ast, offset=""): string = - result = offset & "(" & (if node.recursive: "^" else: "") & node.name & node.kind.toString() - - if node.children.nBl and not node.recursive: - result &= "\n" - for child in node.children: - result &= printAst(child, offset & " ") - result &= offset & ")\n" - else: - result &= ")\n" - -proc parseLisp*(tree: string): ref Ast = - tokenize(tree) - - return readFromTokens() diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index e7e8d75..9310cf7 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -2,7 +2,6 @@ import sets, strformat, strutils import ".."/treesitter/[api, c, cpp] import ".."/globals -import "."/getters template withCodeAst*(code: string, mode: string, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code diff --git a/tests/getheader.nims b/tests/getheader.nims index 916abcf..e5d964c 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -14,7 +14,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = doAssert outp.contains(output), outp var - cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\" -d:checkAbi" + cmd = "nim c -f --hints:off -d:checkAbi" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" sshcmd = " -r libssh2.nim" diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 172f6ae..09343bc 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -17,12 +17,12 @@ cOverride: SOCKET = object when not libssh2Static: - cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") + cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: - cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") + cImport(libssh2Path, recurse = true, flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} diff --git a/tests/rsa.nim b/tests/rsa.nim index a4aa64b..f5c343c 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -39,7 +39,7 @@ cOverride: cImport(@[ basePath / "rsa.h", basePath / "err.h", -], recurse = true, flags = "-f:ast2 -s -c " & FLAGS) +], recurse = true, flags = "-s -c " & FLAGS) {.passL: cryptoLPath.} From 63a75bce4ee36ad2e09ff4957408c7f49e319c2a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 12:24:15 -0500 Subject: [PATCH 196/255] No more includes in build --- nimterop/build.nim | 129 +++++----------------------------- nimterop/build/ccompiler.nim | 2 + nimterop/build/conan.nim | 6 ++ nimterop/build/getheader.nim | 7 +- nimterop/build/jbb.nim | 6 ++ nimterop/build/nimconf.nim | 6 ++ nimterop/build/shell.nim | 85 ++++++++++++++++------ nimterop/build/tools.nim | 22 +++--- nimterop/cimport.nim | 1 - nimterop/globals.nim | 20 ++++-- nimterop/paths.nim | 2 +- nimterop/setup.nim | 3 +- nimterop/toast.nim | 10 ++- nimterop/toastlib/getters.nim | 3 +- nimterop/treesitter/cpp.nim | 3 +- 15 files changed, 144 insertions(+), 161 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 672cb58..3a73f56 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,130 +1,39 @@ -import hashes, osproc, sets, strformat, strutils - when not defined(TOAST): - import macros, tables - -import os except findExe, sleep + import os except findExe, sleep +else: + import os export extractFilename, `/` -# build specific debug since we cannot import globals (yet) -var - gDebug* = false - gDebugCT* {.compileTime.} = false - gNimExe* = "" - # Misc helpers - -proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = - result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) - if not noQuote: - result = result.quoteShell - -proc getCurrentNimCompiler*(): string = - when nimvm: - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - else: - result = gNimExe - -template fixOutDir() {.dirty, used.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - -proc compareVersions*(ver1, ver2: string): int = - ## Compare two version strings x.y.z and return -1, 0, 1 - ## - ## ver1 < ver2 = -1 - ## ver1 = ver2 = 0 - ## ver1 > ver2 = 1 - let - ver1seq = ver1.replace("-", "").split('.') - ver2seq = ver2.replace("-", "").split('.') - for i in 0 ..< ver1seq.len: - let - p1 = ver1seq[i] - p2 = if i < ver2seq.len: ver2seq[i] else: "0" - - try: - let - h1 = p1.parseHexInt() - h2 = p2.parseHexInt() - - if h1 < h2: return -1 - elif h1 > h2: return 1 - except ValueError: - if p1 < p2: return -1 - elif p1 > p2: return 1 - -proc fixCmd(cmd: string): string = - when defined(Windows): - # Replace 'cd d:\abc' with 'd: && cd d:\abc` - var filteredCmd = cmd - if cmd.toLower().startsWith("cd"): - var - colonIndex = cmd.find(":") - driveLetter = cmd.substr(colonIndex-1, colonIndex) - if (driveLetter[0].isAlphaAscii() and - driveLetter[1] == ':' and - colonIndex == 4): - filteredCmd = &"{driveLetter} && {cmd}" - result = "cmd /c " & filteredCmd - elif defined(posix): - result = cmd - else: - doAssert false +import "."/build/misc +export misc # Nim cfg file related functionality -include "."/build/nimconf - -proc getNimteropCacheDir(): string = - # Get location to cache all nimterop artifacts - result = getNimcacheDir() / "nimterop" +import "."/build/nimconf +export nimconf # Functionality shelled out to external executables -include "."/build/shell - -proc getProjectCacheDir*(name: string, forceClean = true): string = - ## Get a cache directory where all nimterop artifacts can be stored - ## - ## Projects can use this location to download source code and build binaries - ## that can be then accessed by multiple apps. This is created under the - ## per-user Nim cache directory. - ## - ## Use `name` to specify the subdirectory name for a project. - ## - ## `forceClean` is enabled by default and effectively deletes the folder - ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows - ## any project to start out with a clean cache dir on a forced build. - ## - ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same - ## `name` when `forceClean = true` else checked out source might get deleted - ## at the wrong time during build. - ## - ## E.g. - ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` - ## - ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so - ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. - result = getNimteropCacheDir() / name - - if forceClean and compileOption("forceBuild"): - echo "# Removing " & result - rmDir(result) +import "."/build/shell +export shell # C compiler support -include "."/build/ccompiler +import "."/build/ccompiler +export ccompiler when not defined(TOAST): # configure, cmake, make support - include "."/build/tools + import "."/build/tools + export tools # Conan.io support - include "."/build/conan + import "."/build/conan + export conan # Julia BinaryBuilder.org support - include "."/build/jbb + import "."/build/jbb + export jbb # getHeader support - include "."/build/getheader + import "."/build/getheader + export getheader diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim index c33c94a..550e1b2 100644 --- a/nimterop/build/ccompiler.nim +++ b/nimterop/build/ccompiler.nim @@ -1,5 +1,7 @@ import os, strformat, strutils +import "."/shell + proc getCompilerMode*(path: string): string = ## Determines a target language mode from an input filename, if one is not already specified. let file = path.splitFile() diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 3430752..31e7f26 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -1,5 +1,7 @@ import os, strformat, strutils, tables +import "."/[ccompiler, misc, nimconf, shell] + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal else: @@ -77,6 +79,10 @@ proc jsonGet(url: string): JsonNode = discard rmFile(file) +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + proc `==`*(pkg1, pkg2: ConanPackage): bool = ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 216b599..8bb13fd 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -1,4 +1,9 @@ -import macros, os, strutils, tables +import macros, strformat, strutils, tables + +import os except findExe + +import ".."/globals +import "."/[ccompiler, conan, jbb, nimconf, shell, tools] var gDefines {.compileTime.} = initTable[string, string]() diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 15bc578..cb48a27 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,5 +1,7 @@ import json, os, strformat, strutils, tables +import "."/[ccompiler, nimconf, shell] + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal @@ -27,6 +29,10 @@ var # Reuse dependencies already downloaded gJBBRequires {.compileTime.}: Table[string, JBBPackage] +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index c87a60f..70e47a3 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,5 +1,7 @@ import json, os, osproc, sets, strformat, strutils +import "."/misc + when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): import std/compilesettings @@ -227,3 +229,7 @@ proc getOutDir*(projectDir = ""): string = let cfg = getNimConfig(projectDir) result = cfg.outDir + +proc getNimteropCacheDir*(): string = + ## Get location to cache all nimterop artifacts + result = getNimcacheDir() / "nimterop" diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index bc07285..7f84db3 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -1,15 +1,25 @@ -import os, strformat, strutils +import hashes, osproc, sets, strformat, strutils -proc sleep*(milsecs: int) = - ## Sleep at compile time - let - cmd = - when defined(Windows): - "cmd /c timeout " - else: - "sleep " +when not defined(TOAST): + import os except findExe, sleep +else: + import os - discard gorgeEx(cmd & $(milsecs / 1000)) +import "."/[misc, nimconf] + +when not defined(TOAST): + proc sleep*(milsecs: int) = + ## Sleep at compile time + let + cmd = + when defined(Windows): + "cmd /c timeout " + else: + "sleep " + + discard gorgeEx(cmd & $(milsecs / 1000)) +else: + export sleep proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = @@ -73,20 +83,23 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & "\nresult:\n" & result.output -proc findExe*(exe: string): string = - ## Find the specified executable using the `which`/`where` command - supported - ## at compile time - var - cmd = - when defined(Windows): - "where " & exe - else: - "which " & exe +when not defined(TOAST): + proc findExe*(exe: string): string = + ## Find the specified executable using the `which`/`where` command - supported + ## at compile time + var + cmd = + when defined(Windows): + "where " & exe + else: + "which " & exe - (output, ret) = execAction(cmd, die = false) + (output, ret) = execAction(cmd, die = false) - if ret == 0: - return output.splitLines()[0].strip().sanitizePath + if ret == 0: + return output.splitLines()[0].strip().sanitizePath +else: + export findExe proc mkDir*(dir: string) = ## Create a directory at compile time @@ -465,3 +478,31 @@ proc getNumProcs*(): string = execAction("sysctl -n hw.ncpu").output.strip() else: "1" + +proc getProjectCacheDir*(name: string, forceClean = true): string = + ## Get a cache directory where all nimterop artifacts can be stored + ## + ## Projects can use this location to download source code and build binaries + ## that can be then accessed by multiple apps. This is created under the + ## per-user Nim cache directory. + ## + ## Use `name` to specify the subdirectory name for a project. + ## + ## `forceClean` is enabled by default and effectively deletes the folder + ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows + ## any project to start out with a clean cache dir on a forced build. + ## + ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same + ## `name` when `forceClean = true` else checked out source might get deleted + ## at the wrong time during build. + ## + ## E.g. + ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` + ## + ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so + ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. + result = getNimteropCacheDir() / name + + if forceClean and compileOption("forceBuild"): + echo "# Removing " & result + rmDir(result) diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index c6a4283..5be1f4b 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -1,20 +1,16 @@ -import os, strformat, strutils +import strformat, strutils -type - BuildType* = enum - btAutoconf, btCmake +import os except findExe - BuildStatus = object - built: bool - buildPath: string - error: string +import ".."/globals +import "."/[misc, shell] proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") - when nimvm: - if gDebugCT: echo str + when defined(TOAST): + if gState.debug: echo str else: - if gDebug: echo str + if gStateCT.debug: echo str proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other @@ -208,7 +204,7 @@ proc make*(path, check: string, flags = "", regex = false) = doAssert findFile(check, path, regex = regex).len != 0, "make failed" -proc buildWithCmake(outdir, flags: string): BuildStatus = +proc buildWithCmake*(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if fileExists(outdir / "CMakeLists.txt"): if findExe("cmake").len != 0: @@ -238,7 +234,7 @@ proc buildWithCmake(outdir, flags: string): BuildStatus = else: result.buildPath = outdir -proc buildWithAutoConf(outdir, flags: string): BuildStatus = +proc buildWithAutoConf*(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if findExe("bash").len != 0: for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index c81c35b..38a75e7 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -376,7 +376,6 @@ proc cSearchPath*(path: string): string {.compileTime.}= proc cDebug*() {.compileTime.} = ## Enable debug messages and display the generated Nim code gStateCT.debug = true - build.gDebugCT = true proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development diff --git a/nimterop/globals.nim b/nimterop/globals.nim index fa8f8de..e8117c8 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -7,7 +7,7 @@ when defined(TOAST): import compiler/[ast, idents, modulegraphs, options] - import "."/treesitter/api + # import "."/treesitter/api type Feature* = enum @@ -78,6 +78,21 @@ type pluginSource*: string # `cPlugin()` generated code to write to plugin file from searchDirs*: seq[string] # `cSearchPath()` added directories for header search + BuildType* = enum + btAutoconf, btCmake + + BuildStatus* = object + built*: bool + buildPath*: string + error*: string + +when nimvm: + var + gStateCT* {.compileTime, used.} = new(State) +else: + var + gState*: State + when defined(TOAST): const gAtoms* {.used.} = @[ @@ -119,9 +134,6 @@ when defined(TOAST): template decho*(args: varargs[string, `$`]): untyped = if gState.debug: gecho join(args, "").getCommented() -else: - var - gStateCT* {.compileTime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/paths.nim b/nimterop/paths.nim index fa245b3..b336d46 100644 --- a/nimterop/paths.nim +++ b/nimterop/paths.nim @@ -1,6 +1,6 @@ import os -import "."/build +import "."/build/shell const cacheDir* = getProjectCacheDir("nimterop", forceClean = false) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 3f16dd7..998d9e9 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -1,6 +1,7 @@ import os, strutils -import "."/[build, paths] +import "."/[paths] +import "."/build/[shell] proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ diff --git a/nimterop/toast.nim b/nimterop/toast.nim index b7a993a..e888e30 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,10 +2,12 @@ import os, osproc, sets, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[build, globals] +import "."/[globals] import "."/toastlib/[ast2, getters, tshelp] +import "."/build/[ccompiler, misc] + proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -54,7 +56,7 @@ proc main( ) = # Setup global state with arguments - var gState = State( + gState = State( convention: convention, debug: debug, defines: defines, @@ -76,10 +78,6 @@ proc main( symOverride: symOverride ) - # Set gDebug in build.nim - build.gDebug = gState.debug - build.gNimExe = gState.nim - # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 0c1b784..c3f48ae 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -2,7 +2,8 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import ".."/[build, globals, plugin] +import ".."/[globals, plugin] +import ".."/build/[ccompiler, misc, nimconf, shell] const gReserved = """ addr and as asm diff --git a/nimterop/treesitter/cpp.nim b/nimterop/treesitter/cpp.nim index 2fe3128..4f5be17 100644 --- a/nimterop/treesitter/cpp.nim +++ b/nimterop/treesitter/cpp.nim @@ -1,6 +1,7 @@ import strutils, os -import ".."/[build, setup, paths] +import ".."/[setup, paths] +import ".."/build/shell static: treesitterCppSetup() From a0413f1595be040c57bc3d75cea1679e5d671d71 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 13:17:25 -0500 Subject: [PATCH 197/255] Use g/decho in build --- nimterop/build/conan.nim | 10 +++--- nimterop/build/getheader.nim | 10 +++--- nimterop/build/jbb.nim | 3 +- nimterop/build/misc.nim | 62 ++++++++++++++++++++++++++++++++++++ nimterop/build/nimconf.nim | 3 +- nimterop/build/shell.nim | 23 ++++++------- nimterop/build/tools.nim | 37 +++++++++------------ nimterop/cimport.nim | 18 +++++------ nimterop/globals.nim | 29 +++++++++++++---- nimterop/toast.nim | 3 +- nimterop/toastlib/tshelp.nim | 3 -- 11 files changed, 135 insertions(+), 66 deletions(-) create mode 100644 nimterop/build/misc.nim diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 31e7f26..792de01 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -1,11 +1,11 @@ -import os, strformat, strutils, tables +import json, os, strformat, strutils, tables + +import ".."/globals import "."/[ccompiler, misc, nimconf, shell] when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal -else: - import json type ConanPackage* = ref object @@ -158,7 +158,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if channel.len != 0: query &= "/" & channel - echo &"# Searching Conan.io for latest version of {name}" + gecho &"# Searching Conan.io for latest version of {name}" let j1 = jsonGet(conanSearchUrl % ["query", query]) @@ -397,7 +397,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" - echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io" + gecho &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 8bb13fd..6972ccc 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -476,9 +476,9 @@ macro getHeader*( {.passL: `ldeps`.join(" ").} static: - echo "# Including library " & lpath + gecho "# Including library " & lpath if `ldeps`.len != 0: - echo "# Including dependencies " & `ldeps`.join(" ") + gecho "# Including dependencies " & `ldeps`.join(" ") else: const `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath @@ -498,7 +498,7 @@ macro getHeader*( ldeps[i] = ldeptgt # Copy downloaded dependencies to `libdir` if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + gecho "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` ldeps else: ldeps @@ -507,8 +507,8 @@ macro getHeader*( when not useStd: # Copy downloaded shared libraries to `libdir` if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + gecho "# Copying " & `lpath`.extractFilename() & " to " & `libdir` cpFile(lpath, `lpath`) - echo "# Including library " & `lpath` + gecho "# Including library " & `lpath` ) diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index cb48a27..0fb163f 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,5 +1,6 @@ import json, os, strformat, strutils, tables +import ".."/globals import "."/[ccompiler, nimconf, shell] when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -198,7 +199,7 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = else: "" path = outdir / pkg.name - echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" + gecho &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" downloadUrl(pkg.url, path, quiet = true) pkg.findJBBLibs(path) diff --git a/nimterop/build/misc.nim b/nimterop/build/misc.nim new file mode 100644 index 0000000..c6c0e48 --- /dev/null +++ b/nimterop/build/misc.nim @@ -0,0 +1,62 @@ +import os, strutils + +when defined(Windows): + import strformat + +import ".."/globals + +proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = + result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) + if not noQuote: + result = result.quoteShell + +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gState.nim + +proc compareVersions*(ver1, ver2: string): int = + ## Compare two version strings x.y.z and return -1, 0, 1 + ## + ## ver1 < ver2 = -1 + ## ver1 = ver2 = 0 + ## ver1 > ver2 = 1 + let + ver1seq = ver1.replace("-", "").split('.') + ver2seq = ver2.replace("-", "").split('.') + for i in 0 ..< ver1seq.len: + let + p1 = ver1seq[i] + p2 = if i < ver2seq.len: ver2seq[i] else: "0" + + try: + let + h1 = p1.parseHexInt() + h2 = p2.parseHexInt() + + if h1 < h2: return -1 + elif h1 > h2: return 1 + except ValueError: + if p1 < p2: return -1 + elif p1 > p2: return 1 + +proc fixCmd*(cmd: string): string = + when defined(Windows): + # Replace 'cd d:\abc' with 'd: && cd d:\abc` + var filteredCmd = cmd + if cmd.toLower().startsWith("cd"): + var + colonIndex = cmd.find(":") + driveLetter = cmd.substr(colonIndex-1, colonIndex) + if (driveLetter[0].isAlphaAscii() and + driveLetter[1] == ':' and + colonIndex == 4): + filteredCmd = &"{driveLetter} && {cmd}" + result = "cmd /c " & filteredCmd + elif defined(posix): + result = cmd + else: + doAssert false diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 70e47a3..3bbe521 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,5 +1,6 @@ import json, os, osproc, sets, strformat, strutils +import ".."/globals import "."/misc when nimvm: @@ -42,7 +43,7 @@ proc getJson(projectDir: string): JsonNode = try: result = parseJson(dump) except JsonParsingError as e: - echo "# Failed to parse `nim dump` output: " & e.msg + gecho "# Failed to parse `nim dump` output: " & e.msg proc getOsCacheDir(): string = # OS default cache directory diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 7f84db3..a5d49de 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -5,6 +5,7 @@ when not defined(TOAST): else: import os +import ".."/globals import "."/[misc, nimconf] when not defined(TOAST): @@ -228,7 +229,7 @@ proc extractZip*(zipfile, outdir: string, quiet = false) = "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" if not quiet: - echo "# Extracting " & zipfile + gecho "# Extracting " & zipfile discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") proc extractTar*(tarfile, outdir: string, quiet = false) = @@ -262,7 +263,7 @@ proc extractTar*(tarfile, outdir: string, quiet = false) = doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath if not quiet: - echo "# Extracting " & tarfile + gecho "# Extracting " & tarfile discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -279,7 +280,7 @@ proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = if not (ext in archives and fileExists(filePath)): if not quiet: - echo "# Downloading " & file + gecho "# Downloading " & file mkDir(outdir) var cmd = findExe("curl") if cmd.len != 0: @@ -302,12 +303,12 @@ proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory - echo "# Resetting " & outdir + gecho "# Resetting " & outdir let cmd = &"cd {outdir.sanitizePath} && git reset --hard" while execAction(cmd).output.contains("Permission denied"): sleep(1000) - echo "# Retrying ..." + gecho "# Retrying ..." proc gitCheckout*(file, outdir: string) = ## Checkout the specified `file` in the git repository at `outdir` @@ -315,12 +316,12 @@ proc gitCheckout*(file, outdir: string) = ## This effectively resets all changes in the file and can be ## used to undo any changes that were made to source files to enable ## successful wrapping with `cImport()` or `c2nImport()`. - echo "# Resetting " & file + gecho "# Resetting " & file let file2 = file.relativePath outdir let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" while execAction(cmd).output.contains("Permission denied"): sleep(500) - echo "# Retrying ..." + gecho "# Retrying ..." proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory @@ -343,7 +344,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false mkDir(outdir) if not quiet: - echo "# Setting up Git repo: " & url + gecho "# Setting up Git repo: " & url discard execAction(&"cd {outdirQ} && git init .") discard execAction(&"cd {outdirQ} && git remote add origin {url}") @@ -360,12 +361,12 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false if checkout.len != 0: if not quiet: - echo "# Checking out " & checkout + gecho "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 3) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: if not quiet: - echo "# Pulling repository" + gecho "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) proc gitTags*(outdir: string): seq[string] = @@ -504,5 +505,5 @@ proc getProjectCacheDir*(name: string, forceClean = true): string = result = getNimteropCacheDir() / name if forceClean and compileOption("forceBuild"): - echo "# Removing " & result + gecho "# Removing " & result rmDir(result) diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index 5be1f4b..c164ffd 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -5,13 +5,6 @@ import os except findExe import ".."/globals import "."/[misc, shell] -proc echoDebug(str: string) = - let str = "\n# " & str.strip().replace("\n", "\n# ") - when defined(TOAST): - if gState.debug: echo str - else: - if gStateCT.debug: echo str - proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -30,18 +23,18 @@ proc configure*(path, check: string, flags = "") = if (path / check).fileExists(): return - echo "# Configuring " & path + gecho "# Configuring " & path if not fileExists(path / "configure"): for i in @["autogen.sh", "build" / "autogen.sh"]: if fileExists(path / i): - echo "# Running autogen.sh" + gecho "# Running autogen.sh" when defined(unix): - echoDebug execAction( + decho execAction( &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output else: - echoDebug execAction( + decho execAction( &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output break @@ -49,14 +42,14 @@ proc configure*(path, check: string, flags = "") = if not fileExists(path / "configure"): for i in @["configure.ac", "configure.in"]: if fileExists(path / i): - echo "# Running autoreconf" + gecho "# Running autoreconf" - echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output + decho execAction(&"cd {path.sanitizePath} && autoreconf -fi").output break if fileExists(path / "configure"): - echo "# Running configure " & flags + gecho "# Running configure " & flags when defined(unix): var @@ -67,7 +60,7 @@ proc configure*(path, check: string, flags = "") = if flags.len != 0: cmd &= &" {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert (path / check).fileExists(), "Configure failed" @@ -156,15 +149,15 @@ proc cmake*(path, check, flags: string) = if (path / check).fileExists(): return - echo "# Running cmake " & flags - echo "# Path: " & path + gecho "# Running cmake " & flags + gecho "# Path: " & path mkDir(path) let cmd = &"cd {path.sanitizePath} && cmake {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert (path / check).fileExists(), "cmake failed" @@ -184,8 +177,8 @@ proc make*(path, check: string, flags = "", regex = false) = if findFile(check, path, regex = regex).len != 0: return - echo "# Running make " & flags - echo "# Path: " & path + gecho "# Running make " & flags + gecho "# Path: " & path var cmd = findExe("make") @@ -200,7 +193,7 @@ proc make*(path, check: string, flags = "", regex = false) = if flags.len != 0: cmd &= &" {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert findFile(check, path, regex = regex).len != 0, "make failed" @@ -219,7 +212,7 @@ proc buildWithCmake*(outdir, flags: string): BuildStatus = elif uname.contains("mingw"): gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" else: - echo "Unsupported system: " & uname + gecho "Unsupported system: " & uname else: gen = "MinGW Makefiles".quoteShell else: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 38a75e7..edfdeb9 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -246,8 +246,8 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = gStateCT.symOverride.add name - if gStateCT.debug and names.nBl: - echo "# Overriding " & names.join(" ") + if names.nBl: + decho "Overriding " & names.join(" ") proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = ## Similar to `cOverride() `_, this macro allows @@ -412,7 +412,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped = {.passC: `str`.} if gStateCT.debug: - echo result.repr & "\n" + gecho result.repr & "\n" proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to @@ -442,7 +442,7 @@ macro cIncludeDir*(dir: static string): untyped = result.add quote do: {.passC: `str`.} if gStateCT.debug: - echo result.repr + gecho result.repr proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search @@ -545,7 +545,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = result.add stmt.parseStmt() if gStateCT.debug: - echo result.repr + gecho result.repr macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = @@ -565,7 +565,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl if gStateCT.pluginSourcePath.Bl: cPluginHelper(gStateCT.pluginSource) - echo "# Importing " & fullpaths.join(", ").sanitizePath + gecho "# Importing " & fullpaths.join(", ").sanitizePath let output = getToast(fullpaths, recurse, dynlib, mode, flags) @@ -576,7 +576,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gStateCT.overrides = "" if gStateCT.debug: - echo output + gecho output try: let body = parseStmt(output) @@ -661,7 +661,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let fullpath = findPath(filename) - echo "# Importing " & fullpath & " with c2nim" + gecho "# Importing " & fullpath & " with c2nim" let output = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) @@ -700,7 +700,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) if gStateCT.debug: - echo nimout + gecho nimout try: let body = parseStmt(nimout) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index e8117c8..a89bac4 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,4 +1,4 @@ -import tables +import strutils, tables when defined(TOAST): import sets, sequtils, strutils @@ -124,16 +124,31 @@ when defined(TOAST): Status* = enum success, unknown, error - # Redirect output to file when required - template gecho*(args: string) = - if gState.outputHandle.isNil: +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +# Redirect output to file when required +template gecho*(args: string) = + when defined(TOAST): + when nimvm: echo args else: - gState.outputHandle.writeLine(args) + if gState.outputHandle.isNil: + echo args + else: + gState.outputHandle.writeLine(args) + else: + echo args - template decho*(args: varargs[string, `$`]): untyped = +template decho*(args: varargs[string, `$`]): untyped = + let + str = join(args, "").getCommented() + when defined(TOAST): if gState.debug: - gecho join(args, "").getCommented() + gecho str + else: + if gStateCT.debug: + echo str template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index e888e30..6fe5180 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -124,8 +124,7 @@ proc main( doAssert gState.outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}" - if gState.debug: - echo &"# Writing output to {outputFile}\n" + decho &"# Writing output to {outputFile}\n" if source.nBl: # Print source after preprocess or Nim output diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 9310cf7..3b7c2cc 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -30,9 +30,6 @@ template withCodeAst*(code: string, mode: string, body: untyped): untyped = defer: tree.tsTreeDelete() -proc getCommented*(str: string): string = - "\n# " & str.strip().replace("\n", "\n# ") - proc isNil*(node: TSNode): bool = node.tsNodeIsNull() From 85a4365e3cef38e6490d3ff591e33fee9d2167a9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 00:10:13 -0500 Subject: [PATCH 198/255] Adjust git checkout if changed, tree-sitter version bump --- nimterop/build/shell.nim | 18 ++++++++++++++---- nimterop/globals.nim | 4 +++- nimterop/setup.nim | 6 +++--- nimterop/toastlib/ast2.nim | 5 ++++- nimterop/toastlib/exprparser.nim | 2 +- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index a5d49de..53c0b5f 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -323,6 +323,12 @@ proc gitCheckout*(file, outdir: string) = sleep(500) gecho "# Retrying ..." +proc gitAtCheckout*(outdir, checkout: string): bool = + ## Check if specified git repository is checked out to the specified + ## commit hash, tag or branch + result = checkout in execAction( + &"cd {outdir.sanitizePath} && git log --decorate --no-color -n 1 --format=oneline").output + proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## @@ -334,13 +340,17 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false ## `checkout` is the git tag, branch or commit hash to checkout once ## the repository is downloaded. This allows for pinning to a specific ## version of the code. - if dirExists(outdir/".git"): - gitReset(outdir) - return - let outdirQ = outdir.sanitizePath + if dirExists(outdir/".git"): + gitReset(outdir) + if checkout.nBl and not gitAtCheckout(outdir, checkout): + gecho &"# Updating repository to checkout {checkout}" + discard execAction( + &"cd {outdirQ} && git clean -fxd && git fetch && git checkout {checkout}", retry = 3) + return + mkDir(outdir) if not quiet: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index a89bac4..c831589 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -111,7 +111,9 @@ when defined(TOAST): "bitwise_expression", "shift_expression", "math_expression", - "escape_sequence" + "escape_sequence", + "binary_expression", + "unary_expression" ].toHashSet() gEnumVals* {.used.} = @[ diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 998d9e9..a0940d2 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -7,7 +7,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.5") +""", "0.15.14") gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ *.c @@ -37,7 +37,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.15.3") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -49,7 +49,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.15.1") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 0016e29..783b6e4 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1099,7 +1099,10 @@ proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode = ncount = if not afdecl.isNil: # Pointer to function pointer - afdecl[0].getXCount("abstract_pointer_declarator") + if afdecl[0].getName() == "abstract_parenthesized_declarator": + afdecl[0][0].getXCount("abstract_pointer_declarator") + else: + afdecl[0].getXCount("abstract_pointer_declarator") else: node.getAtom().tsNodeParent().getPtrCount(reverse = true) diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index 28b49d3..13b1a98 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -535,7 +535,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # once we upgrade of "math_expression", "logical_expression", "relational_expression", "bitwise_expression", "equality_expression", "binary_expression", - "shift_expression": + "shift_expression", "unary_expression": # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b # Output -> # typeof(a)(a == typeof(a)(b)) From f42ba492e3b4649e9fb0e18c095464669aed463e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 00:59:42 -0500 Subject: [PATCH 199/255] Bump to latest tree-sitter --- nimterop/setup.nim | 11 +- nimterop/treesitter/api.nim | 306 ++++++++++++++++++++++------------ nimterop/treesitter/tsgen.nim | 9 +- 3 files changed, 199 insertions(+), 127 deletions(-) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index a0940d2..6da0bbe 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -7,12 +7,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.14") - - gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ -*.c -*.h -""") +""", "0.16.8") let tbase = cacheDir / "treesitter" / "lib" @@ -37,7 +32,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.3") +""", "0.16.1") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -49,7 +44,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.1") +""", "v0.16.0") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 09e328d..116bc10 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -12,56 +12,70 @@ const sourcePath = cacheDir / "treesitter" / "lib" when defined(Linux): {.passC: "-std=c11".} -{.passC: "-DUTF8PROC_STATIC".} {.passC: "-I$1" % (sourcePath / "include").} {.passC: "-I$1" % (sourcePath / "src").} -{.passC: "-I$1" % (sourcePath / ".." / ".." / "utf8proc").} {.compile: sourcePath / "src" / "lib.c".} ### Generated below -{.hint[ConvFromXtoItselfNotNeeded]: off.} +{.push hint[ConvFromXtoItselfNotNeeded]: off.} +{.pragma: impapiHdr, header: sourcePath / "include" / "tree_sitter" / "api.h".} defineEnum(TSInputEncoding) defineEnum(TSSymbolType) defineEnum(TSLogType) +defineEnum(TSQueryPredicateStepType) +defineEnum(TSQueryError) + const - headerapi {.used.} = sourcePath / "include" / "tree_sitter" / "api.h" - TREE_SITTER_LANGUAGE_VERSION* = 9 - TSInputEncodingUTF8* = 0.TSInputEncoding - TSInputEncodingUTF16* = 1.TSInputEncoding - TSSymbolTypeRegular* = 0.TSSymbolType - TSSymbolTypeAnonymous* = 1.TSSymbolType - TSSymbolTypeAuxiliary* = 2.TSSymbolType - TSLogTypeParse* = 0.TSLogType - TSLogTypeLex* = 1.TSLogType + TREE_SITTER_LANGUAGE_VERSION* = 11 + TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION* = 9 + TSInputEncodingUTF8* = (0).TSInputEncoding + TSInputEncodingUTF16* = (TSInputEncodingUTF8 + 1).TSInputEncoding + TSSymbolTypeRegular* = (0).TSSymbolType + TSSymbolTypeAnonymous* = (TSSymbolTypeRegular + 1).TSSymbolType + TSSymbolTypeAuxiliary* = (TSSymbolTypeAnonymous + 1).TSSymbolType + TSLogTypeParse* = (0).TSLogType + TSLogTypeLex* = (TSLogTypeParse + 1).TSLogType + TSQueryPredicateStepTypeDone* = (0).TSQueryPredicateStepType + TSQueryPredicateStepTypeCapture* = (TSQueryPredicateStepTypeDone + 1).TSQueryPredicateStepType + TSQueryPredicateStepTypeString* = (TSQueryPredicateStepTypeCapture + 1).TSQueryPredicateStepType + TSQueryErrorNone* = (0).TSQueryError + TSQueryErrorSyntax* = (TSQueryErrorNone + 1).TSQueryError + TSQueryErrorNodeType* = (TSQueryErrorSyntax + 1).TSQueryError + TSQueryErrorField* = (TSQueryErrorNodeType + 1).TSQueryError + TSQueryErrorCapture* = (TSQueryErrorField + 1).TSQueryError + type - TSSymbol* = uint16 - TSLanguage* = object - TSParser* = object - TSTree* = object - TSPoint* {.importc, header: headerapi, bycopy.} = object + TSSymbol* {.importc, impapiHdr.} = uint16 + TSFieldId* {.importc, impapiHdr.} = uint16 + TSLanguage* {.importc, impapiHdr, incompleteStruct.} = object + TSParser* {.importc, impapiHdr, incompleteStruct.} = object + TSTree* {.importc, impapiHdr, incompleteStruct.} = object + TSQuery* {.importc, impapiHdr, incompleteStruct.} = object + TSQueryCursor* {.importc, impapiHdr, incompleteStruct.} = object + TSPoint* {.bycopy, importc, impapiHdr.} = object row*: uint32 column*: uint32 - TSRange* {.importc, header: headerapi, bycopy.} = object + TSRange* {.bycopy, importc, impapiHdr.} = object start_point*: TSPoint end_point*: TSPoint start_byte*: uint32 end_byte*: uint32 - TSInput* {.importc, header: headerapi, bycopy.} = object + TSInput* {.bycopy, importc, impapiHdr.} = object payload*: pointer read*: proc (payload: pointer; byte_index: uint32; position: TSPoint; - bytes_read: ptr uint32): cstring {.nimcall.} + bytes_read: ptr uint32): cstring {.cdecl.} encoding*: TSInputEncoding - TSLogger* {.importc, header: headerapi, bycopy.} = object + TSLogger* {.bycopy, importc, impapiHdr.} = object payload*: pointer - log*: proc (payload: pointer; a1: TSLogType; a2: cstring) {.nimcall.} + log*: proc (payload: pointer; a2: TSLogType; a3: cstring) {.cdecl.} - TSInputEdit* {.importc, header: headerapi, bycopy.} = object + TSInputEdit* {.bycopy, importc, impapiHdr.} = object start_byte*: uint32 old_end_byte*: uint32 new_end_byte*: uint32 @@ -69,106 +83,176 @@ type old_end_point*: TSPoint new_end_point*: TSPoint - TSNode* {.importc, header: headerapi, bycopy.} = object + TSNode* {.bycopy, importc, impapiHdr.} = object context*: array[4, uint32] id*: pointer tree*: ptr TSTree - TSTreeCursor* {.importc, header: headerapi, bycopy.} = object + TSTreeCursor* {.bycopy, importc, impapiHdr.} = object tree*: pointer id*: pointer context*: array[2, uint32] -proc ts_parser_new*(): ptr TSParser {.importc, header: headerapi.} -proc ts_parser_delete*(a1: ptr TSParser) {.importc, header: headerapi.} -proc ts_parser_language*(a1: ptr TSParser): ptr TSLanguage {.importc, header: headerapi.} -proc ts_parser_set_language*(a1: ptr TSParser; a2: ptr TSLanguage): bool {.importc, - header: headerapi.} -proc ts_parser_logger*(a1: ptr TSParser): TSLogger {.importc, header: headerapi.} -proc ts_parser_set_logger*(a1: ptr TSParser; a2: TSLogger) {.importc, header: headerapi.} -proc ts_parser_print_dot_graphs*(a1: ptr TSParser; a2: cint) {.importc, - header: headerapi.} -proc ts_parser_halt_on_error*(a1: ptr TSParser; a2: bool) {.importc, header: headerapi.} -proc ts_parser_parse*(a1: ptr TSParser; a2: ptr TSTree; a3: TSInput): ptr TSTree {.importc, - header: headerapi.} -proc ts_parser_parse_string*(a1: ptr TSParser; a2: ptr TSTree; a3: cstring; a4: uint32): ptr TSTree {. - importc, header: headerapi.} -proc ts_parser_parse_string_encoding*(a1: ptr TSParser; a2: ptr TSTree; a3: cstring; - a4: uint32; a5: TSInputEncoding): ptr TSTree {. - importc, header: headerapi.} -proc ts_parser_enabled*(a1: ptr TSParser): bool {.importc, header: headerapi.} -proc ts_parser_set_enabled*(a1: ptr TSParser; a2: bool) {.importc, header: headerapi.} -proc ts_parser_operation_limit*(a1: ptr TSParser): cuint {.importc, header: headerapi.} -proc ts_parser_set_operation_limit*(a1: ptr TSParser; a2: cuint) {.importc, - header: headerapi.} -proc ts_parser_reset*(a1: ptr TSParser) {.importc, header: headerapi.} -proc ts_parser_set_included_ranges*(a1: ptr TSParser; a2: ptr TSRange; a3: uint32) {. - importc, header: headerapi.} -proc ts_parser_included_ranges*(a1: ptr TSParser; a2: ptr uint32): ptr TSRange {.importc, - header: headerapi.} -proc ts_tree_copy*(a1: ptr TSTree): ptr TSTree {.importc, header: headerapi.} -proc ts_tree_delete*(a1: ptr TSTree) {.importc, header: headerapi.} -proc ts_tree_root_node*(a1: ptr TSTree): TSNode {.importc, header: headerapi.} -proc ts_tree_edit*(a1: ptr TSTree; a2: ptr TSInputEdit) {.importc, header: headerapi.} -proc ts_tree_get_changed_ranges*(a1: ptr TSTree; a2: ptr TSTree; a3: ptr uint32): ptr TSRange {. - importc, header: headerapi.} -proc ts_tree_print_dot_graph*(a1: ptr TSTree; a2: ptr FILE) {.importc, header: headerapi.} -proc ts_tree_language*(a1: ptr TSTree): ptr TSLanguage {.importc, header: headerapi.} -proc ts_node_start_byte*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_start_point*(a1: TSNode): TSPoint {.importc, header: headerapi.} -proc ts_node_end_byte*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_end_point*(a1: TSNode): TSPoint {.importc, header: headerapi.} -proc ts_node_symbol*(a1: TSNode): TSSymbol {.importc, header: headerapi.} -proc ts_node_type*(a1: TSNode): cstring {.importc, header: headerapi.} -proc ts_node_string*(a1: TSNode): cstring {.importc, header: headerapi.} -proc ts_node_eq*(a1: TSNode; a2: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_null*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_named*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_missing*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_has_changes*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_has_error*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_parent*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_child*(a1: TSNode; a2: uint32): TSNode {.importc, header: headerapi.} -proc ts_node_named_child*(a1: TSNode; a2: uint32): TSNode {.importc, header: headerapi.} -proc ts_node_child_count*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_named_child_count*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_next_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_next_named_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_prev_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_prev_named_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_first_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, - header: headerapi.} + TSQueryCapture* {.bycopy, importc, impapiHdr.} = object + node*: TSNode + index*: uint32 + + TSQueryMatch* {.bycopy, importc, impapiHdr.} = object + id*: uint32 + pattern_index*: uint16 + capture_count*: uint16 + captures*: ptr TSQueryCapture + + TSQueryPredicateStep* {.bycopy, importc, impapiHdr.} = object + `type`*: TSQueryPredicateStepType + value_id*: uint32 + +proc ts_parser_new*(): ptr TSParser {.importc, cdecl, impapiHdr.} +proc ts_parser_delete*(parser: ptr TSParser) {.importc, cdecl, impapiHdr.} +proc ts_parser_set_language*(self: ptr TSParser; language: ptr TSLanguage): bool {. + importc, cdecl, impapiHdr.} +proc ts_parser_language*(self: ptr TSParser): ptr TSLanguage {.importc, cdecl, impapiHdr.} +proc ts_parser_set_included_ranges*(self: ptr TSParser; ranges: ptr TSRange; + length: uint32) {.importc, cdecl, impapiHdr.} +proc ts_parser_included_ranges*(self: ptr TSParser; length: ptr uint32): ptr TSRange {. + importc, cdecl, impapiHdr.} +proc ts_parser_parse*(self: ptr TSParser; old_tree: ptr TSTree; input: TSInput): ptr TSTree {. + importc, cdecl, impapiHdr.} +proc ts_parser_parse_string*(self: ptr TSParser; old_tree: ptr TSTree; string: cstring; + length: uint32): ptr TSTree {.importc, cdecl, impapiHdr.} +proc ts_parser_parse_string_encoding*(self: ptr TSParser; old_tree: ptr TSTree; + string: cstring; length: uint32; + encoding: TSInputEncoding): ptr TSTree {. + importc, cdecl, impapiHdr.} +proc ts_parser_reset*(self: ptr TSParser) {.importc, cdecl, impapiHdr.} +proc ts_parser_set_timeout_micros*(self: ptr TSParser; timeout: uint64) {.importc, + cdecl, impapiHdr.} +proc ts_parser_timeout_micros*(self: ptr TSParser): uint64 {.importc, cdecl, impapiHdr.} +proc ts_parser_set_cancellation_flag*(self: ptr TSParser; flag: ptr uint) {.importc, + cdecl, impapiHdr.} +proc ts_parser_cancellation_flag*(self: ptr TSParser): ptr uint {.importc, cdecl, + impapiHdr.} +proc ts_parser_set_logger*(self: ptr TSParser; logger: TSLogger) {.importc, cdecl, + impapiHdr.} +proc ts_parser_logger*(self: ptr TSParser): TSLogger {.importc, cdecl, impapiHdr.} +proc ts_parser_print_dot_graphs*(self: ptr TSParser; file: cint) {.importc, cdecl, + impapiHdr.} +proc ts_parser_halt_on_error*(self: ptr TSParser; halt: bool) {.importc, cdecl, + impapiHdr.} +proc ts_tree_copy*(self: ptr TSTree): ptr TSTree {.importc, cdecl, impapiHdr.} +proc ts_tree_delete*(self: ptr TSTree) {.importc, cdecl, impapiHdr.} +proc ts_tree_root_node*(self: ptr TSTree): TSNode {.importc, cdecl, impapiHdr.} +proc ts_tree_language*(a1: ptr TSTree): ptr TSLanguage {.importc, cdecl, impapiHdr.} +proc ts_tree_edit*(self: ptr TSTree; edit: ptr TSInputEdit) {.importc, cdecl, impapiHdr.} +proc ts_tree_get_changed_ranges*(old_tree: ptr TSTree; new_tree: ptr TSTree; + length: ptr uint32): ptr TSRange {.importc, cdecl, + impapiHdr.} +proc ts_tree_print_dot_graph*(a1: ptr TSTree; a2: File) {.importc, cdecl, impapiHdr.} +proc ts_node_type*(a1: TSNode): cstring {.importc, cdecl, impapiHdr.} +proc ts_node_symbol*(a1: TSNode): TSSymbol {.importc, cdecl, impapiHdr.} +proc ts_node_start_byte*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_start_point*(a1: TSNode): TSPoint {.importc, cdecl, impapiHdr.} +proc ts_node_end_byte*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_end_point*(a1: TSNode): TSPoint {.importc, cdecl, impapiHdr.} +proc ts_node_string*(a1: TSNode): cstring {.importc, cdecl, impapiHdr.} +proc ts_node_is_null*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_named*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_missing*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_extra*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_has_changes*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_has_error*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_parent*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_child*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_child_count*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_named_child*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_named_child_count*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_child_by_field_name*(self: TSNode; field_name: cstring; + field_name_length: uint32): TSNode {.importc, + cdecl, impapiHdr.} +proc ts_node_child_by_field_id*(a1: TSNode; a2: TSFieldId): TSNode {.importc, cdecl, + impapiHdr.} +proc ts_node_next_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_prev_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_next_named_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_prev_named_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_first_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, + impapiHdr.} proc ts_node_first_named_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, - header: headerapi.} + cdecl, impapiHdr.} proc ts_node_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. - importc, header: headerapi.} -proc ts_node_named_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. - importc, header: headerapi.} + importc, cdecl, impapiHdr.} proc ts_node_descendant_for_point_range*(a1: TSNode; a2: TSPoint; a3: TSPoint): TSNode {. - importc, header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_node_named_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. + importc, cdecl, impapiHdr.} proc ts_node_named_descendant_for_point_range*(a1: TSNode; a2: TSPoint; a3: TSPoint): TSNode {. - importc, header: headerapi.} -proc ts_node_edit*(a1: ptr TSNode; a2: ptr TSInputEdit) {.importc, header: headerapi.} -proc ts_tree_cursor_new*(a1: TSNode): TSTreeCursor {.importc, header: headerapi.} -proc ts_tree_cursor_delete*(a1: ptr TSTreeCursor) {.importc, header: headerapi.} -proc ts_tree_cursor_reset*(a1: ptr TSTreeCursor; a2: TSNode) {.importc, - header: headerapi.} -proc ts_tree_cursor_current_node*(a1: ptr TSTreeCursor): TSNode {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_parent*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_next_sibling*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_first_child*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_node_edit*(a1: ptr TSNode; a2: ptr TSInputEdit) {.importc, cdecl, impapiHdr.} +proc ts_node_eq*(a1: TSNode; a2: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_new*(a1: TSNode): TSTreeCursor {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_delete*(a1: ptr TSTreeCursor) {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_reset*(a1: ptr TSTreeCursor; a2: TSNode) {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_current_node*(a1: ptr TSTreeCursor): TSNode {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_current_field_name*(a1: ptr TSTreeCursor): cstring {.importc, + cdecl, impapiHdr.} +proc ts_tree_cursor_current_field_id*(a1: ptr TSTreeCursor): TSFieldId {.importc, + cdecl, impapiHdr.} +proc ts_tree_cursor_goto_parent*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_goto_next_sibling*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_goto_first_child*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} proc ts_tree_cursor_goto_first_child_for_byte*(a1: ptr TSTreeCursor; a2: uint32): int64 {. - importc, header: headerapi.} -proc ts_language_symbol_count*(a1: ptr TSLanguage): uint32 {.importc, - header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_tree_cursor_copy*(a1: ptr TSTreeCursor): TSTreeCursor {.importc, cdecl, + impapiHdr.} +proc ts_query_new*(language: ptr TSLanguage; source: cstring; source_len: uint32; + error_offset: ptr uint32; error_type: ptr TSQueryError): ptr TSQuery {. + importc, cdecl, impapiHdr.} +proc ts_query_delete*(a1: ptr TSQuery) {.importc, cdecl, impapiHdr.} +proc ts_query_pattern_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_capture_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_string_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_start_byte_for_pattern*(a1: ptr TSQuery; a2: uint32): uint32 {.importc, + cdecl, impapiHdr.} +proc ts_query_predicates_for_pattern*(self: ptr TSQuery; pattern_index: uint32; + length: ptr uint32): ptr TSQueryPredicateStep {. + importc, cdecl, impapiHdr.} +proc ts_query_capture_name_for_id*(a1: ptr TSQuery; id: uint32; length: ptr uint32): cstring {. + importc, cdecl, impapiHdr.} +proc ts_query_string_value_for_id*(a1: ptr TSQuery; id: uint32; length: ptr uint32): cstring {. + importc, cdecl, impapiHdr.} +proc ts_query_disable_capture*(a1: ptr TSQuery; a2: cstring; a3: uint32) {.importc, + cdecl, impapiHdr.} +proc ts_query_cursor_new*(): ptr TSQueryCursor {.importc, cdecl, impapiHdr.} +proc ts_query_cursor_delete*(a1: ptr TSQueryCursor) {.importc, cdecl, impapiHdr.} +proc ts_query_cursor_exec*(a1: ptr TSQueryCursor; a2: ptr TSQuery; a3: TSNode) {.importc, + cdecl, impapiHdr.} +proc ts_query_cursor_set_byte_range*(a1: ptr TSQueryCursor; a2: uint32; a3: uint32) {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_set_point_range*(a1: ptr TSQueryCursor; a2: TSPoint; a3: TSPoint) {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_next_match*(a1: ptr TSQueryCursor; match: ptr TSQueryMatch): bool {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_remove_match*(a1: ptr TSQueryCursor; id: uint32) {.importc, cdecl, + impapiHdr.} +proc ts_query_cursor_next_capture*(a1: ptr TSQueryCursor; match: ptr TSQueryMatch; + capture_index: ptr uint32): bool {.importc, cdecl, + impapiHdr.} +proc ts_language_symbol_count*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} proc ts_language_symbol_name*(a1: ptr TSLanguage; a2: TSSymbol): cstring {.importc, - header: headerapi.} -proc ts_language_symbol_for_name*(a1: ptr TSLanguage; a2: cstring): TSSymbol {.importc, - header: headerapi.} + cdecl, impapiHdr.} +proc ts_language_symbol_for_name*(self: ptr TSLanguage; string: cstring; + length: uint32; is_named: bool): TSSymbol {.importc, + cdecl, impapiHdr.} +proc ts_language_field_count*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} +proc ts_language_field_name_for_id*(a1: ptr TSLanguage; a2: TSFieldId): cstring {. + importc, cdecl, impapiHdr.} +proc ts_language_field_id_for_name*(a1: ptr TSLanguage; a2: cstring; a3: uint32): TSFieldId {. + importc, cdecl, impapiHdr.} proc ts_language_symbol_type*(a1: ptr TSLanguage; a2: TSSymbol): TSSymbolType {. - importc, header: headerapi.} -proc ts_language_version*(a1: ptr TSLanguage): uint32 {.importc, header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_language_version*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} +{.pop.} diff --git a/nimterop/treesitter/tsgen.nim b/nimterop/treesitter/tsgen.nim index 484483c..ee2c8e0 100644 --- a/nimterop/treesitter/tsgen.nim +++ b/nimterop/treesitter/tsgen.nim @@ -5,14 +5,7 @@ import os import nimterop/[cimport, paths] -cPlugin: - import strutils - - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - if "_CRT" in sym.name: - sym.name = sym.name.strip(chars={'_'}) - static: cDebug() -cImport(cacheDir / "treesitter" /"lib" / "include" / "tree_sitter" / "api.h") +cImport(cacheDir / "treesitter" / "lib" / "include" / "tree_sitter" / "api.h", flags = "-E_ -c") From e3e1d80a55777b695db393fc440226093044472b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 10:00:12 -0500 Subject: [PATCH 200/255] Add minitest for nim CI --- CHANGES.md | 2 ++ nimterop.nimble | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 93bb8af..6748521 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - The `dynlib` command line parameter to `toast` and `cImport()` can also be the path to a shared library (dll|so|dylib) in place of a Nim const string containing the path. This allows for the traditional use case of passing `"xxxLPath"` to `cImport()` as well as simply passing the path to the library on the command line as is. This allows the creation of standalone cached wrappers as well as the usage of the `--check` and the `--stub` functionality that `toast` provides via `cImport()`. +- `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. + ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index 7be89c6..101805e 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -45,6 +45,11 @@ task btd, "build toast": task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") +task minitest, "Test for Nim CI": + exec "nim c -f -d:danger nimterop/toast" + exec "nim c -f -d:checkAbi -r tests/tast2.nim" + exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" + task test, "Test": rmFile("tests/timeit.txt") From 4ece07969d9aea6e9d8966b64fa445477115483a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 27 Jun 2020 12:28:22 -0500 Subject: [PATCH 201/255] Bump dep versions --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 101805e..bf9f520 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -9,7 +9,7 @@ bin = @["nimterop/toast"] installDirs = @["nimterop"] # Dependencies -requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" +requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.0.0" import nimterop/docs import os From f543124d7e2bec7f7ca8a47b70d88438a96443ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 29 Jun 2020 21:47:00 -0500 Subject: [PATCH 202/255] Fix #125 - no more types import --- nimterop/all.nim | 2 +- nimterop/cimport.nim | 5 +-- nimterop/enumtype.nim | 42 ++++++++++++++++++ nimterop/globals.nim | 5 ++- nimterop/toastlib/ast2.nim | 11 +++-- nimterop/toastlib/comphelp.nim | 2 +- nimterop/toastlib/exprparser.nim | 2 +- nimterop/toastlib/getters.nim | 41 +++++++++++++++++- nimterop/treesitter/api.nim | 3 +- nimterop/types.nim | 74 -------------------------------- 10 files changed, 99 insertions(+), 88 deletions(-) create mode 100644 nimterop/enumtype.nim delete mode 100644 nimterop/types.nim diff --git a/nimterop/all.nim b/nimterop/all.nim index 6be6d11..20ab4e8 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -2,4 +2,4 @@ The following modules are available to users of Nimterop. ]## -import "."/[docs, cimport, build, types, plugin] +import "."/[build, cimport, docs, plugin] diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index edfdeb9..b5c62a2 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,8 +17,7 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import "."/[build, globals, paths, types] -export types +import "."/[build, globals, paths] proc interpPath(dir: string): string= # TODO: more robust: needs a DirSep after "$projpath" @@ -153,7 +152,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" # see https://github.com/nimterop/nimterop/issues/69 (result, ret) = execAction(cmd, die = false, cache = (not gStateCT.nocache), - cacheKey = getCacheValue(fullpaths)) + cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths)) doAssert ret == 0, getToastError(result) macro cOverride*(body): untyped = diff --git a/nimterop/enumtype.nim b/nimterop/enumtype.nim new file mode 100644 index 0000000..3c20b79 --- /dev/null +++ b/nimterop/enumtype.nim @@ -0,0 +1,42 @@ +import macros + +macro defineEnum(typ: untyped): untyped = + result = newNimNode(nnkStmtList) + + # Enum mapped to distinct cint + result.add quote do: + type `typ`* = distinct cint + + for i in ["+", "-", "*", "div", "mod", "shl", "shr", "or", "and", "xor", "<", "<=", "==", ">", ">="]: + let + ni = newIdentNode(i) + typout = if i[0] in "<=>": newIdentNode("bool") else: typ # comparisons return bool + if i[0] == '>': # cannot borrow `>` and `>=` from templates + let + nopp = if i.len == 2: newIdentNode("<=") else: newIdentNode("<") + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` = `nopp`(y, x) + proc `ni`*(x: cint, y: `typ`): `typout` = `nopp`(y, x) + proc `ni`*(x, y: `typ`): `typout` = `nopp`(y, x) + else: + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` {.borrow.} + proc `ni`*(x: cint, y: `typ`): `typout` {.borrow.} + proc `ni`*(x, y: `typ`): `typout` {.borrow.} + result.add quote do: + proc `ni`*(x: `typ`, y: int): `typout` = `ni`(x, y.cint) + proc `ni`*(x: int, y: `typ`): `typout` = `ni`(x.cint, y) + + let + divop = newIdentNode("/") # `/`() + dlrop = newIdentNode("$") # `$`() + notop = newIdentNode("not") # `not`() + result.add quote do: + proc `divop`*(x, y: `typ`): `typ` = `typ`((x.float / y.float).cint) + proc `divop`*(x: `typ`, y: cint): `typ` = `divop`(x, `typ`(y)) + proc `divop`*(x: cint, y: `typ`): `typ` = `divop`(`typ`(x), y) + proc `divop`*(x: `typ`, y: int): `typ` = `divop`(x, y.cint) + proc `divop`*(x: int, y: `typ`): `typ` = `divop`(x.cint, y) + + proc `dlrop`*(x: `typ`): string {.borrow.} + proc `notop`*(x: `typ`): `typ` {.borrow.} diff --git a/nimterop/globals.nim b/nimterop/globals.nim index c831589..d1e22a6 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -54,7 +54,7 @@ type constIdentifiers*: HashSet[string] # Const names for enum casting identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or - # the user provided override is blank + # the user provided override is blank # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode @@ -70,6 +70,9 @@ type # Controls whether or not the current expression # should validate idents against currently defined idents skipIdentValidation*: bool + + # Top level header for wrapper output - include imported types, pragmas and other info + wrapperHeader*: string else: # cimport.nim specific compile*: seq[string] # `cCompile()` list of files already processed diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 783b6e4..0147463 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1481,6 +1481,11 @@ proc addEnum(gState: State, node: TSNode) = if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) + if gEnumMacro.nBl: + # Add enum generation macro once + gState.wrapperHeader &= gEnumMacro + gEnumMacro = "" + proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # @@ -1822,10 +1827,7 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = proc initNim*(gState: State) = # Initialize for parseNim() one time - gecho """import nimterop/types - -{.push hint[ConvFromXtoItselfNotNeeded]: off.} -""" + gState.wrapperHeader = "{.push hint[ConvFromXtoItselfNotNeeded]: off.}\n" # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() @@ -1876,6 +1878,7 @@ proc printNim*(gState: State) = tree.add gState.varSection tree.add gState.procSection + gecho gState.wrapperHeader gecho tree.renderTree() gecho "{.pop.}" \ No newline at end of file diff --git a/nimterop/toastlib/comphelp.nim b/nimterop/toastlib/comphelp.nim index d404c42..642fbbd 100644 --- a/nimterop/toastlib/comphelp.nim +++ b/nimterop/toastlib/comphelp.nim @@ -95,7 +95,7 @@ proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): result.name = gState.getIdentifier(result.origname, kind, parent) if result.name.nBl: if kind == nskType: - result.name = result.name.getType() + result.name = gState.getType(result.name, parent) result.info = gState.getLineInfo(node) proc getPtrType*(str: string): string = diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index 13b1a98..a77dbf9 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -580,7 +580,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = of "sized_type_specifier", "primitive_type", "type_identifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc - let ty = getType(node.val) + let ty = gState.getType(node.val, parent = node.getName()) if ty.len > 0: # If ty is not empty, one of C's builtin types has been found result = gState.getExprIdent(ty, nskType, parent=node.getName()) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index c3f48ae..43a4f6f 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -29,7 +29,13 @@ yield""".split(Whitespace).toHashSet() # Types related +const + # Enum macro read from file - written into wrapper when required + gEnumMacroConst = staticRead(currentSourcePath.parentDir().parentDir() / "enumtype.nim") + var + gEnumMacro* = gEnumMacroConst + gTypeMap* = { # char "char": "cchar", @@ -107,13 +113,40 @@ var "long double": "clongdouble", # Misc Nim types - "Bool": "bool" + "Bool": "bool", + "ptrdiff_t": "ByteAddress" }.toTable() # Nim type names that shouldn't need to be wrapped again gTypeMapValues* = toSeq(gTypeMap.values).toHashSet() -proc getType*(str: string): string = + # Types to import from C/Nim if used in wrapper + gTypeImport* = { + "time_t": """ +import std/time_t as std_time_t +type time_t* = Time +""", + + "time64_t": """ +import std/time_t as std_time64_t +type time64_t* = Time +""", + + "wchar_t": """ +when defined(cpp): + # http://www.cplusplus.com/reference/cwchar/wchar_t/ + # In C++, wchar_t is a distinct fundamental type (and thus it is + # not defined in nor any other header). + type wchar_t* {.importc.} = object +else: + type wchar_t* {.importc, header:"".} = object +""", + + "va_list": """ +type va_list* {.importc, header:"".} = object +"""}.toTable() + +proc getType*(gState: State, str, parent: string): string = if str == "void": return "object" @@ -121,6 +154,10 @@ proc getType*(str: string): string = if gTypeMap.hasKey(result): result = gTypeMap[result] + elif parent.nBl and gTypeImport.hasKey(result) and not gState.identifierNodes.hasKey(result): + # Include C/Nim type imports once if a field/param and not already declared + gState.wrapperHeader &= "\n" & gTypeImport[result] + gTypeImport.del result # Identifier related diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 116bc10..753178c 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -2,7 +2,8 @@ import strutils, os -import ".."/[setup, paths, types] +include ".."/enumtype +import ".."/[paths, setup] static: treesitterSetup() diff --git a/nimterop/types.nim b/nimterop/types.nim deleted file mode 100644 index 38f95d1..0000000 --- a/nimterop/types.nim +++ /dev/null @@ -1,74 +0,0 @@ -# see https://github.com/nimterop/nimterop/issues/79 - -import std/time_t as time_t_temp -type - time_t* = time_t_temp.Time - time64_t* = time_t_temp.Time - -when defined(cpp): - # http://www.cplusplus.com/reference/cwchar/wchar_t/ - # In C++, wchar_t is a distinct fundamental type (and thus it is - # not defined in nor any other header). - type - wchar_t* {.importc.} = object -else: - type - wchar_t* {.importc, header:"".} = object - -type - ptrdiff_t* = ByteAddress - -type - va_list* {.importc, header:"".} = object - -template enumOp*(op, typ, typout) = - proc op*(x: typ, y: cint): typout {.borrow.} - proc op*(x: cint, y: typ): typout {.borrow.} - proc op*(x, y: typ): typout {.borrow.} - - proc op*(x: typ, y: int): typout = op(x, y.cint) - proc op*(x: int, y: typ): typout = op(x.cint, y) - -template defineEnum*(typ) = - # Create a `distinct cint` type for C enums since Nim enums - # need to be in order and cannot have duplicates. - type - typ* = distinct cint - - # Enum operations allowed - enumOp(`+`, typ, typ) - enumOp(`-`, typ, typ) - enumOp(`*`, typ, typ) - enumOp(`<`, typ, bool) - enumOp(`<=`, typ, bool) - enumOp(`==`, typ, bool) - enumOp(`div`, typ, typ) - enumOp(`mod`, typ, typ) - - # These don't work with `enumOp()` for some reason - proc `shl`*(x: typ, y: cint): typ {.borrow.} - proc `shl`*(x: cint, y: typ): typ {.borrow.} - proc `shl`*(x, y: typ): typ {.borrow.} - - proc `shr`*(x: typ, y: cint): typ {.borrow.} - proc `shr`*(x: cint, y: typ): typ {.borrow.} - proc `shr`*(x, y: typ): typ {.borrow.} - - proc `or`*(x: typ, y: cint): typ {.borrow.} - proc `or`*(x: cint, y: typ): typ {.borrow.} - proc `or`*(x, y: typ): typ {.borrow.} - - proc `and`*(x: typ, y: cint): typ {.borrow.} - proc `and`*(x: cint, y: typ): typ {.borrow.} - proc `and`*(x, y: typ): typ {.borrow.} - - proc `xor`*(x: typ, y: cint): typ {.borrow.} - proc `xor`*(x: cint, y: typ): typ {.borrow.} - proc `xor`*(x, y: typ): typ {.borrow.} - - proc `/`*(x, y: typ): typ = - return (x.float / y.float).cint.typ - proc `/`*(x: typ, y: cint): typ = `/`(x, y.typ) - proc `/`*(x: cint, y: typ): typ = `/`(x.typ, y) - - proc `$`*(x: typ): string {.borrow.} From a28c3e7cb4f6da157667b921ab725b92f6774512 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 14:08:09 -0500 Subject: [PATCH 203/255] Improve errors - don't point to macros.nim --- CHANGES.md | 6 ++ nimterop.nimble | 2 +- nimterop/build/shell.nim | 14 +++-- nimterop/cimport.nim | 124 +++++++++++++++++++-------------------- nimterop/toast.nim | 9 ++- 5 files changed, 84 insertions(+), 71 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6748521..3488252 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. +### Other improvements + +- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125](i125) (since v0.6.1) + +- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. + ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index bf9f520..ebc5f0a 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.0" +version = "0.6.1" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 53c0b5f..2b29734 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -23,7 +23,8 @@ else: export sleep proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = + cacheKey = "", onRetry: proc() = nil, + onError: proc(output: string, err: int) = nil): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time ## ## Checks if command exits successfully before returning. If not, an @@ -34,6 +35,8 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, ## `die = false` - return on errors ## `cache = true` - cache results unless cleared with -f ## `cacheKey` - key to create unique cache entry + ## `onRetry()` - proc to call before retrying + ## `onError(output, err)` - proc to call on error let ccmd = fixCmd(cmd) @@ -80,9 +83,12 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, onRetry() sleep(500) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) - elif die: - doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & - "\nresult:\n" & result.output + else: + if not onError.isNil: + onError(result.output, result.ret) + + doAssert not die, "Command failed: " & $result.ret & "\ncmd: " & ccmd & + "\nresult:\n" & result.output when not defined(TOAST): proc findExe*(exe: string): string = diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index b5c62a2..6e0a80f 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,7 +17,8 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import "."/[build, globals, paths] +import "."/[globals, paths] +import "."/build/[ccompiler, misc, nimconf, shell] proc interpPath(dir: string): string= # TODO: more robust: needs a DirSep after "$projpath" @@ -90,35 +91,30 @@ proc getToastError(output: string): string = if result.Bl: result = "\n\n" & output -proc getNimCheckError(output: string): tuple[tmpFile, errors: string] = - let - hash = output.hash().abs() - - result.tmpFile = getProjectCacheDir("failed", forceClean = false) / "nimterop_" & $hash & ".nim" - - if not fileExists(result.tmpFile) or gStateCT.nocache or compileOption("forceBuild"): - mkDir(result.tmpFile.parentDir()) - writeFile(result.tmpFile, output) - - doAssert fileExists(result.tmpFile), "Failed to write to cache dir: " & result.tmpFile - +proc getNimCheckError(nimFile: string) = let (check, _) = execAction( - &"{getCurrentNimCompiler()} check {result.tmpFile.sanitizePath}", + &"{getCurrentNimCompiler()} check {nimFile.sanitizePath}", die = false ) - result.errors = "\n\n" & check + doAssert false, &"\n\n{check}\n\n" & + "Codegen limitation or error - review 'nim check' output above generated for " & nimFile proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "", mode = "c", flags = "", noNimout = false): string = var - ret = 0 cmd = when defined(Windows): "cmd /c " else: "" + ext = "h" + + let + toastExe = toastExePath() + # see https://github.com/nimterop/nimterop/issues/69 + cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths) - let toastExe = toastExePath() doAssert fileExists(toastExe), "toast not compiled: " & toastExe.sanitizePath & " make sure 'nimble build' or 'nimble install' built it" + cmd &= &"{toastExe} --preprocess -m:{mode}" if recurse: @@ -147,13 +143,29 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" if gStateCT.pluginSourcePath.nBl: cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}" + ext = "nim" + for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - # see https://github.com/nimterop/nimterop/issues/69 - (result, ret) = execAction(cmd, die = false, cache = (not gStateCT.nocache), - cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths)) - doAssert ret == 0, getToastError(result) + # Generate filename for toast output + let + hash = (cmd & cacheKey).hash().abs() + cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash + + result = cachePath.addFileExt(ext) + + if not fileExists(result) or compileOption("forceBuild"): + let + dir = cachePath.parentDir() + if not dirExists(dir): + mkDir(dir) + + cmd.add &" -o {result.sanitizePath}" + + var + (_, ret) = execAction(cmd, die = false) + doAssert ret == 0, getToastError(result.readFile()) macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not @@ -567,7 +579,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho "# Importing " & fullpaths.join(", ").sanitizePath let - output = getToast(fullpaths, recurse, dynlib, mode, flags) + nimFile = getToast(fullpaths, recurse, dynlib, mode, flags) # Reset plugin and overrides for next cImport if gStateCT.overrides.nBl: @@ -575,16 +587,12 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gStateCT.overrides = "" if gStateCT.debug: - gecho output + gecho nimFile.readFile() try: - let body = parseStmt(output) - - result.add body + result.add parseStmt("include " & nimFile.changeFileExt("")) except: - let - (tmpFile, errors) = getNimCheckError(output) - doAssert false, errors & "\n\nNimterop codegen limitation or error - review 'nim check' output above generated for " & tmpFile + getNimCheckError(nimFile) macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = @@ -663,49 +671,37 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: gecho "# Importing " & fullpath & " with c2nim" let - output = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - hash = output.hash().abs() - hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" - npath = hpath[0 .. hpath.rfind('.')] & "nim" + hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) + nimFile = hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() - if not fileExists(hpath) or gStateCT.nocache or compileOption("forceBuild"): - mkDir(hpath.parentDir()) - writeFile(hpath, output) + if not fileExists(nimFile) or compileOption("forceBuild"): + var + cmd = when defined(Windows): "cmd /c " else: "" + cmd &= &"c2nim {hFile} --header:{header}" - doAssert fileExists(hpath), "Unable to write temporary header file: " & hpath + if dynlib.nBl: + cmd.add &" --dynlib:{dynlib}" + if mode.contains("cpp"): + cmd.add " --cpp" + if flags.nBl: + cmd.add &" {flags}" - var - cmd = when defined(Windows): "cmd /c " else: "" - cmd &= &"c2nim {hpath} --header:{header}" + for i in gStateCT.defines: + cmd.add &" --assumedef:{i.quoteShell}" - if dynlib.nBl: - cmd.add &" --dynlib:{dynlib}" - if mode.contains("cpp"): - cmd.add " --cpp" - if flags.nBl: - cmd.add &" {flags}" + let + (c2nimout, ret) = execAction(cmd) + if ret != 0: + rmFile(nimFile) + doAssert false, "\n\nc2nim codegen limitation or error - " & c2nimout - for i in gStateCT.defines: - cmd.add &" --assumedef:{i.quoteShell}" - - let - (c2nimout, ret) = execAction(cmd, cache = not gStateCT.nocache, - cacheKey = getCacheValue(hpath)) - - doAssert ret == 0, "\n\nc2nim codegen limitation or error - " & c2nimout - - var - nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) + nimFile.writeFile(&"const {header} = \"{fullpath}\"\n\n" & readFile(nimFile)) if gStateCT.debug: - gecho nimout + gecho nimFile.readFile() try: - let body = parseStmt(nimout) - - result.add body + result.add parseStmt("include " & nimFile.changeFileExt("")) except: - let - (tmpFile, errors) = getNimCheckError(nimout) - doAssert false, errors & "\n\nc2nim codegen limitation or error - review 'nim check' output above generated for " & tmpFile + getNimCheckError(nimFile) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 6fe5180..fbaa960 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -8,6 +8,10 @@ import "."/toastlib/[ast2, getters, tshelp] import "."/build/[ccompiler, misc] +var + # Output generated before main() is called + preMainOut = "" + proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -129,6 +133,7 @@ proc main( if source.nBl: # Print source after preprocess or Nim output if gState.pnim: + gecho preMainOut gState.initNim() for src in source: gState.process(src.expandSymlinkAbs()) @@ -187,7 +192,7 @@ proc mergeParams(cmdNames: seq[string], cmdLine = commandLineParams()): seq[stri # https://github.com/c-blake/cligen/issues/149 for param in cmdLine: if param.fileExists() and param.splitFile().ext == ".cfg": - echo &"# Loading flags from '{param}'" + preMainOut &= &"# Loading flags from '{param}'\n" for line in param.readFile().splitLines(): let line = line.strip() @@ -197,7 +202,7 @@ proc mergeParams(cmdNames: seq[string], cmdLine = commandLineParams()): seq[stri result.add param if result.len != 0 and "-h" notin result and "--help" notin result: - echo &"""# Generated @ {$now()} + preMainOut &= &"""# Generated @ {$now()} # Command line: # {getAppFilename()} {result.join(" ")} """ From b22b23620da088f8728802199bb1a8c0f5999ec6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 15:43:09 -0500 Subject: [PATCH 204/255] Fix Windows include issue --- nimterop/cimport.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 6e0a80f..109b172 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -154,6 +154,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash result = cachePath.addFileExt(ext) + when defined(Windows): + result = result.replace(DirSep, '/') if not fileExists(result) or compileOption("forceBuild"): let @@ -590,7 +592,10 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho nimFile.readFile() try: - result.add parseStmt("include " & nimFile.changeFileExt("")) + let + nimFileNode = newStrLitNode(nimFile.changeFileExt("")) + result.add quote do: + include `nimFileNode` except: getNimCheckError(nimFile) @@ -702,6 +707,9 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: gecho nimFile.readFile() try: - result.add parseStmt("include " & nimFile.changeFileExt("")) + let + nimFileNode = newStrLitNode(nimFile.changeFileExt("")) + result.add quote do: + include `nimFileNode` except: getNimCheckError(nimFile) From 2aef7c99b36bbc9db9bcbcd62c06c6a5e2d4d0c0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 22:33:10 -0500 Subject: [PATCH 205/255] Fix #127 - write wrapper to file --- CHANGES.md | 14 +++-- nimterop.nimble | 30 +++++++--- nimterop/cimport.nim | 135 ++++++++++++++++++++----------------------- tests/tast2.nim | 6 +- 4 files changed, 100 insertions(+), 85 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3488252..43f1d34 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,13 +15,13 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 ### Breaking changes - The legacy algorithm has been removed as promised. `ast2` is now the default and wrappers no longer need to explicitly specify `-f:ast2` in order to use it. -- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) +- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154][i154] - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. @@ -33,11 +33,13 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. +- `cImport()` can now write the generated wrapper output to a user-defined file with the `nimFile` param. [#127][i127] (since v0.6.1) + ### Other improvements -- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125](i125) (since v0.6.1) +- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) -- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. +- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) ## Version 0.5.0 @@ -50,7 +52,7 @@ Version 0.6.0 of Nimterop will make `ast2` the default backend and the legacy al See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 +https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 ### Breaking changes @@ -108,6 +110,8 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 [i54]: https://github.com/nimterop/nimterop/issues/54 [i74]: https://github.com/nimterop/nimterop/issues/74 [i76]: https://github.com/nimterop/nimterop/issues/76 +[i125]: https://github.com/nimterop/nimterop/issues/125 +[i127]: https://github.com/nimterop/nimterop/issues/127 [i137]: https://github.com/nimterop/nimterop/issues/137 [i147]: https://github.com/nimterop/nimterop/issues/147 [i148]: https://github.com/nimterop/nimterop/issues/148 diff --git a/nimterop.nimble b/nimterop.nimble index ebc5f0a..628ef40 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -50,14 +50,10 @@ task minitest, "Test for Nim CI": exec "nim c -f -d:checkAbi -r tests/tast2.nim" exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" -task test, "Test": - rmFile("tests/timeit.txt") - - buildTimeitTask() - buildToastTask() - +task basic, "Basic tests": execTest "tests/tast2.nim" execTest "tests/tast2.nim", "-d:NOHEADER" + execTest "tests/tast2.nim", "-d:NOHEADER -d:WRAPPED" execTest "tests/tnimterop_c.nim" execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-H\"" @@ -65,6 +61,7 @@ task test, "Test": execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast tests/toast.cfg tests/include/toast.h" +task wrapper, "Wrapper tests": execTest "tests/tpcre.nim" when defined(Linux): @@ -79,12 +76,29 @@ task test, "Test": execTest "tests/tsoloud.nim" execTest "tests/tsoloud.nim", "-d:FLAGS=\"-H\"" - # getHeader tests +task getheader, "getHeader tests": withDir("tests"): exec "nim e getheader.nims" - if not existsEnv("APPVEYOR"): + +task package, "Wrapper package tests": + if not existsEnv("APPVEYOR"): + withDir("tests"): exec "nim e wrappers.nims" +task test, "Test": + rmFile("tests/timeit.txt") + + buildTimeitTask() + buildToastTask() + + basicTask() + + wrapperTask() + + getheaderTask() + + packageTask() + docsTask() echo readFile("tests/timeit.txt") diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 109b172..d992c04 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -70,6 +70,9 @@ proc walkDirImpl(indir, inext: string, file=true): seq[string] = if ret == 0: result = output.splitLines() +proc fixRelFile(file: string): string = + if file.isAbsolute(): file else: getProjectDir() / file + proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() @@ -102,7 +105,7 @@ proc getNimCheckError(nimFile: string) = "Codegen limitation or error - review 'nim check' output above generated for " & nimFile proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "", - mode = "c", flags = "", noNimout = false): string = + mode = "c", flags = "", outFile = "", noNimout = false): string = var cmd = when defined(Windows): "cmd /c " else: "" ext = "h" @@ -148,18 +151,17 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - # Generate filename for toast output - let - hash = (cmd & cacheKey).hash().abs() - cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash + result = if outFile.nBl: fixRelFile(outFile) else: + # Generate filename for toast output if not specified + getNimteropCacheDir() / "toastCache" / "nimterop_" & + ($(cmd & cacheKey).hash().abs()).addFileExt(ext) - result = cachePath.addFileExt(ext) when defined(Windows): result = result.replace(DirSep, '/') if not fileExists(result) or compileOption("forceBuild"): let - dir = cachePath.parentDir() + dir = result.parentDir() if not dirExists(dir): mkDir(dir) @@ -171,9 +173,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not - ## accurate, it may be required to hand wrap them. Define them in a - ## `cOverride() `_ macro block so that Nimterop uses - ## these definitions instead. + ## accurate, it may be required to hand wrap them. Define them in a `cOverride()` + ## macro block so that Nimterop uses these definitions instead. ## ## For example: ## @@ -194,9 +195,9 @@ macro cOverride*(body): untyped = ## cOverride: ## proc svGetCallerInfo(fileName: var cstring; lineNumber: var cint) ## - ## Using the `cOverride() `_ block, nimterop - ## can be instructed to use this definition of `svGetCallerInfo()` instead. - ## This works for procs, consts and types. + ## Using the `cOverride()` block, nimterop can be instructed to use this + ## definition of `svGetCallerInfo()` instead. This works for procs, consts + ## and types. ## ## `cOverride()` only affects the next `cImport()` call. This is because any ## recognized symbols get overridden in place and any remaining symbols get @@ -263,11 +264,10 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = decho "Overriding " & names.join(" ") proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = - ## Similar to `cOverride() `_, this macro allows - ## filtering out symbols not of interest from the generated output. + ## Similar to `cOverride()`, this macro allows filtering out symbols not of + ## interest from the generated output. ## - ## `cSkipSymbol() `_ only affects calls to - ## `cImport() `_ that follow it. + ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. runnableExamples: static: cSkipSymbol @["proc1", "Type2"] gStateCT.symOverride.add skips @@ -291,11 +291,9 @@ proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") gStateCT.pluginSourcePath = path macro cPlugin*(body): untyped = - ## When `cOverride() `_ and - ## `cSkipSymbol() `_ - ## are not adequate, the `cPlugin() `_ macro can be used - ## to customize the generated Nim output. The following callbacks are available at - ## this time. + ## When `cOverride()` and `cSkipSymbol()` are not adequate, the `cPlugin()` + ## macro can be used to customize the generated Nim output. The following + ## callbacks are available at this time. ## ## .. code-block:: nim ## @@ -325,9 +323,7 @@ macro cPlugin*(body): untyped = ## `macros` and `nimterop/plugins` are implicitly imported to provide access to standard ## plugin facilities. ## - ## `cPlugin() `_ only affects calls to - ## `cImport() `_ that - ## follow it. + ## `cPlugin()` only affects calls to `cImport()` that follow it. runnableExamples: cPlugin: import strutils @@ -367,14 +363,10 @@ macro cPluginPath*(path: static[string]): untyped = proc cSearchPath*(path: string): string {.compileTime.}= ## Get full path to file or directory `path` in search path configured - ## using `cAddSearchDir() `_ and - ## `cAddStdDir() `_. + ## using `cAddSearchDir()` and `cAddStdDir()`. ## ## This can be used to locate files or directories that can be passed onto - ## `cCompile() `_, - ## `cIncludeDir() `_ and - ## `cImport() `_. - + ## `cCompile()`, `cIncludeDir()` and `cImport()`. result = findPath(path, fail = false) if result.Bl: var found = false @@ -394,21 +386,17 @@ proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development ## ## If files included by header being processed by - ## `cImport() `_ - ## change and affect the generated content, they will be ignored and the cached - ## value will continue to be used . Use `cDisableCaching() `_ - ## to avoid this scenario during development. + ## `cImport()` change and affect the generated content, they will be ignored + ## and the cached value will continue to be used . Use `cDisableCaching()` to + ## avoid this scenario during development. ## - ## `nim -f` was broken prior to 0.19.4 but can also be used to flush the cached content. - + ## `nim -f` can also be used to flush the cached content. gStateCT.nocache = true macro cDefine*(name: static string, val: static string = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if - ## called within `cImport() `_ - ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-DXXX".}` - + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` result = newNimNode(nnkStmtList) var str = name @@ -429,7 +417,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped = proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to - ## `cSearchPath() `_. + ## `cSearchPath()`. runnableExamples: import nimterop/paths, os static: @@ -441,10 +429,8 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = macro cIncludeDir*(dir: static string): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if - ## called within `cImport() `_ - ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-IXXX".}`. - + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. var dir = interpPath(dir) result = newNimNode(nnkStmtList) @@ -459,7 +445,7 @@ macro cIncludeDir*(dir: static string): untyped = proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search - ## path used in calls to `cSearchPath() `_ + ## path used in calls to `cSearchPath()`. runnableExamples: static: cAddStdDir() import os @@ -561,7 +547,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = gecho result.repr macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import multiple headers in one shot ## ## This macro is preferable over multiple individual `cImport()` calls, especially @@ -581,7 +567,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho "# Importing " & fullpaths.join(", ").sanitizePath let - nimFile = getToast(fullpaths, recurse, dynlib, mode, flags) + nimFile = getToast(fullpaths, recurse, dynlib, mode, flags, nimFile) # Reset plugin and overrides for next cImport if gStateCT.overrides.nBl: @@ -600,16 +586,15 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl getNimCheckError(nimFile) macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import all supported definitions from specified header file. Generated ## content is cached in `nimcache` until `filename` changes unless - ## `cDisableCaching() `_ is set. `nim -f` - ## can also be used after Nim v0.19.4 to flush the cache. + ## `cDisableCaching()` is set. `nim -f` can also be used to flush the cache. ## ## `recurse` can be used to generate Nim wrappers from `#include` files ## referenced in `filename`. This is only done for files in the same ## directory as `filename` or in a directory added using - ## `cIncludeDir() `_ + ## `cIncludeDir()`. ## ## `dynlib` can be used to specify the Nim string to use to specify the dynamic ## library to load the imported symbols from. For example: @@ -630,9 +615,9 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## ## cImport("pcre.h", dynlib="dynpcre") ## - ## If `dynlib` is not specified, the C/C++ implementation files can be compiled in - ## with `cCompile() `_, or the - ## `{.passL.}` pragma can be used to specify the static lib to link. + ## If `dynlib` is not specified, the C/C++ implementation files can be compiled + ## in with `cCompile()`, or the `{.passL.}` pragma can be used to specify the + ## static lib to link. ## ## `mode` selects the preprocessor and tree-sitter parser to be used to process ## the header. @@ -641,33 +626,41 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## good example would be `--prefix` and `--suffix` which strip leading and ## trailing strings from identifiers, `_` being quite common. ## + ## `nimFile` is the location where the generated wrapper should get written. + ## By default, the generated wrapper is written to `nimcache` and included from + ## there. `nimFile` makes it possible to write the wrapper to a predetermined + ## location which can then be directly imported into the main application and + ## checked into source control if preferred. Importing the nimterop wrapper with + ## `nimFile` specified still works per usual. If `nimFile` is not an absolute + ## path, it is relative to the project path. + ## ## `cImport()` consumes and resets preceding `cOverride()` calls. `cPlugin()` ## is retained for the next `cImport()` call unless a new `cPlugin()` call is ## defined. return quote do: - cImport(@[`filename`], bool(`recurse`), `dynlib`, `mode`, `flags`) + cImport(@[`filename`], bool(`recurse`), `dynlib`, `mode`, `flags`, `nimFile`) macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import all supported definitions from specified header file using `c2nim` ## - ## Similar to `cImport() `_ - ## but uses `c2nim` to generate the Nim wrapper instead of `toast`. Note that neither - ## `cOverride() `_, `cSkipSymbol() `_ - ## nor `cPlugin() `_ have any impact on `c2nim`. + ## Similar to `cImport()` but uses `c2nim` to generate the Nim wrapper instead + ## of `toast`. Note that neither `cOverride()`, `cSkipSymbol()` nor `cPlugin()` + ## have any impact on `c2nim`. ## - ## `toast` is only used to preprocess the header file and recurse - ## if specified. + ## `toast` is only used to preprocess the header file and `recurse` if specified. ## ## `mode` should be set to `cpp` for c2nim to wrap C++ headers. ## ## `flags` can be used to pass other command line arguments to `c2nim`. ## - ## `nimterop` does not depend on `c2nim` as a `nimble` dependency so it - ## does not get installed automatically. Any wrapper or library that requires this proc - ## needs to install `c2nim` with `nimble install c2nim` or add it as a dependency in - ## its own `.nimble` file. - + ## `nimFile` is the location where the generated wrapper should get written, + ## similar to `cImport()`. + ## + ## `nimterop` does not depend on `c2nim` as a `nimble` dependency so it does not + ## get installed automatically. Any wrapper or library that requires this proc + ## needs to install `c2nim` with `nimble install c2nim` or add it as a dependency + ## in its own `.nimble` file. result = newNimNode(nnkStmtList) let @@ -677,13 +670,13 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - nimFile = hFile.changeFileExt("nim") + nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(nimFile) or compileOption("forceBuild"): var cmd = when defined(Windows): "cmd /c " else: "" - cmd &= &"c2nim {hFile} --header:{header}" + cmd &= &"c2nim {hFile} --header:{header} --out:{nimFile.sanitizePath}" if dynlib.nBl: cmd.add &" --dynlib:{dynlib}" diff --git a/tests/tast2.nim b/tests/tast2.nim index 3f626ae..669ec98 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -38,7 +38,11 @@ cOverride: A1* = A0 cDefine("SOME_CONST=100") -cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags) + +when not defined(WRAPPED): + cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags, nimFile = "tast2wrapped.nim") +else: + import tast2wrapped proc getPragmas(n: NimNode): HashSet[string] = # Find all pragmas in AST, return as "name" or "name:value" in set From a63f67f6850dab10ec0c8784d370f8178edc7bf9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 00:10:16 -0500 Subject: [PATCH 206/255] Minor fixes, improvements --- nimterop/globals.nim | 8 ++-- nimterop/setup.nim | 8 ---- nimterop/toastlib/getters.nim | 83 ++++++++++++++++++----------------- nimterop/treesitter/c.nim | 2 +- nimterop/treesitter/cpp.nim | 2 +- 5 files changed, 48 insertions(+), 55 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d1e22a6..e0b9b8f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -129,7 +129,7 @@ when defined(TOAST): Status* = enum success, unknown, error -proc getCommented*(str: string): string = +template getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") # Redirect output to file when required @@ -147,13 +147,13 @@ template gecho*(args: string) = template decho*(args: varargs[string, `$`]): untyped = let - str = join(args, "").getCommented() + str = join(args, "") when defined(TOAST): if gState.debug: - gecho str + gecho str.getCommented() else: if gStateCT.debug: - echo str + echo str.getCommented() template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 6da0bbe..6da1358 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -34,10 +34,6 @@ src/*.cc src/tree_sitter/parser.h """, "0.16.1") - writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ -const TSLanguage *tree_sitter_c(); -""") - proc treesitterCppSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter-cpp", cacheDir / "treesitter_cpp", """ src/*.h @@ -45,7 +41,3 @@ src/*.c src/*.cc src/tree_sitter/parser.h """, "v0.16.0") - - writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ -const TSLanguage *tree_sitter_cpp(); -""") diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 43a4f6f..534b6fa 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times +import dynlib, macros, os, osproc, sequtils, sets, streams, strformat, strutils, tables, times import regex @@ -124,12 +124,12 @@ var gTypeImport* = { "time_t": """ import std/time_t as std_time_t -type time_t* = Time +type time_t* = std_time_t.Time """, "time64_t": """ import std/time_t as std_time64_t -type time64_t* = Time +type time64_t* = std_time_t.Time """, "wchar_t": """ @@ -286,10 +286,7 @@ proc getCurrentHeader*(fullpath: string): string = proc getPreprocessor*(gState: State, fullpath: string) = var - cmts = if gState.noComments: "" else: "-CC" - cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - - rdata: seq[string] + args: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) @@ -297,50 +294,54 @@ proc getPreprocessor*(gState: State, fullpath: string) = pDir = sfile.expandFilename().parentDir() includeDirs: seq[string] + args.add @["-E", "-dD", getGccModeArg(gState.mode), "-w"] + if not gState.noComments: + args.add "-CC" + for inc in gState.includeDirs: - cmd &= &"-I{inc.sanitizePath} " + args.add &"-I{inc.sanitizePath}" includeDirs.add inc.absolutePath().sanitizePath(noQuote = true) for def in gState.defines: - cmd &= &"-D{def} " + args.add &"-D{def}" # Remove gcc special calls - if defined(posix): - cmd &= "-D__attribute__\\(x\\)= " - else: - cmd &= "-D__attribute__(x)= " - - cmd &= "-D__restrict= -D__extension__= -D__inline__=inline -D__inline=inline " - # https://github.com/tree-sitter/tree-sitter-c/issues/43 - cmd &= "-D_Noreturn= " - - cmd &= &"{fullpath.sanitizePath}" + args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__extension__=", "-D__inline__=inline", + "-D__inline=inline", "-D_Noreturn=", &"{fullpath.sanitizePath}"] # Include content only from file - for line in execAction(cmd).output.splitLines(): - # We want to keep blank lines here for comment processing - if line.len > 1 and line[0 .. 1] == "# ": - start = false - let - saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine or - (DirSep notin saniLine and sfileName in saniLine): - start = true - elif gState.recurse: - if pDir.Bl or pDir in saniLine: + var + p = startProcess(getCompiler(), args = args, options = {poStdErrToStdOut, poUsePath}) + outp = p.outputStream() + line = "" + + # Include content only from file + gState.code = "" + while true: + if outp.readLine(line): + # We want to keep blank lines here for comment processing + if line.len > 1 and line[0] == '#' and line[1] == ' ': + start = false + line = line.sanitizePath(noQuote = true) + if sfile in line or + (DirSep notin line and sfileName in line): start = true - else: - for inc in includeDirs: - if inc in saniLine: - start = true - break - else: - if start: - if "#undef" in line: - continue - rdata.add line - gState.code = rdata.join("\n") + elif gState.recurse: + if pDir.Bl or pDir in line: + start = true + else: + for inc in includeDirs: + if inc in line: + start = true + break + else: + if start: + if "#undef" in line: + continue + gState.code.add line & "\n" + elif not p.running(): break + p.close() # Plugin related diff --git a/nimterop/treesitter/c.nim b/nimterop/treesitter/c.nim index d779bfd..4280b91 100644 --- a/nimterop/treesitter/c.nim +++ b/nimterop/treesitter/c.nim @@ -13,4 +13,4 @@ import "."/api {.compile: srcDir / "parser.c".} -proc treeSitterC*(): ptr TSLanguage {.importc: "tree_sitter_c", header: srcDir / "api.h".} +proc treeSitterC*(): ptr TSLanguage {.importc: "tree_sitter_c".} diff --git a/nimterop/treesitter/cpp.nim b/nimterop/treesitter/cpp.nim index 4f5be17..fd0437e 100644 --- a/nimterop/treesitter/cpp.nim +++ b/nimterop/treesitter/cpp.nim @@ -18,4 +18,4 @@ static: {.compile: srcDir / "parser_cpp.c".} {.compile: srcDir / "scanner.cc".} -proc treeSitterCpp*(): ptr TSLanguage {.importc: "tree_sitter_cpp", header: srcDir / "api.h".} +proc treeSitterCpp*(): ptr TSLanguage {.importc: "tree_sitter_cpp".} From 94acdeb5f1e0576decdb116570d5b2f05af82bf7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 02:29:28 -0500 Subject: [PATCH 207/255] Fix #238 crash --- nimterop/toastlib/ast2.nim | 2 +- nimterop/toastlib/getters.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 0147463..5769893 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1668,7 +1668,7 @@ proc addDecl(gState: State, node: TSNode) = commentNodes: seq[TSNode] for i in start+1 ..< node.len: - if node[i].getName() == "comment": + if node[i].getName() in ["comment", "type_qualifier"]: continue if firstDecl: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 534b6fa..3a71c78 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -129,7 +129,7 @@ type time_t* = std_time_t.Time "time64_t": """ import std/time_t as std_time64_t -type time64_t* = std_time_t.Time +type time64_t* = std_time64_t.Time """, "wchar_t": """ From 1e88a3b42ec6d27c1d924ee9f6840852b1a68797 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 12:41:56 -0500 Subject: [PATCH 208/255] Fix static inline and restrict --- nimterop/toastlib/ast2.nim | 3 ++- nimterop/toastlib/getters.nim | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 5769893..dc4d220 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1703,8 +1703,9 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) commentNodes = gState.getCommentNodes(node) + fdecl = node[start+1].firstChildInTree("function_declarator") - if node[start+1].getName() == "function_declarator": + if not fdecl.isNil: if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 3a71c78..08ba604 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -307,7 +307,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = # Remove gcc special calls # https://github.com/tree-sitter/tree-sitter-c/issues/43 - args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__extension__=", "-D__inline__=inline", + args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__restrict__=", "-D__extension__=", "-D__inline__=inline", "-D__inline=inline", "-D_Noreturn=", &"{fullpath.sanitizePath}"] # Include content only from file From c3f4f3e487ce8bf8bd9e6e9a370a808e00638091 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 20:11:11 -0500 Subject: [PATCH 209/255] Handle tserror in static inline, wrap headers only once --- nimterop/globals.nim | 1 + nimterop/toast.nim | 6 +++++- nimterop/toastlib/ast2.nim | 23 ++++++++++++----------- nimterop/toastlib/getters.nim | 15 ++++++++++----- nimterop/toastlib/tshelp.nim | 9 +++++++++ 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index e0b9b8f..17b9518 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -55,6 +55,7 @@ type identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or # the user provided override is blank + headersProcessed*: HashSet[string] # Headers already processed directly or recursively # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode diff --git a/nimterop/toast.nim b/nimterop/toast.nim index fbaa960..1cda239 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -136,7 +136,11 @@ proc main( gecho preMainOut gState.initNim() for src in source: - gState.process(src.expandSymlinkAbs()) + let + src = src.expandSymlinkAbs() + if src notin gState.headersProcessed: + gState.process(src) + gState.headersProcessed.incl src if gState.pnim: printNim(gState) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index dc4d220..43ec8d3 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -137,7 +137,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = # In case symbol was skipped earlier gState.skippedSyms.excl origname else: - gecho &"# const '{origname}' is duplicate, skipped" + decho &"const '{origname}' is duplicate, skipped" else: gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname @@ -391,7 +391,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: gState.identifierNodes[name] = result else: - gecho &"# {getKeyword(kind)} '{origname}' is duplicate, skipped" + decho &"{getKeyword(kind)} '{origname}' is duplicate, skipped" proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input @@ -1706,6 +1706,9 @@ proc addDef(gState: State, node: TSNode) = fdecl = node[start+1].firstChildInTree("function_declarator") if not fdecl.isNil: + if gState.getNodeError(fdecl): + return + if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: @@ -1716,19 +1719,14 @@ proc processNode(gState: State, node: TSNode): Status = const known = ["preproc_def", "type_definition", "struct_specifier", "union_specifier", "enum_specifier", - "declaration", "function_definition"].toHashSet() + "declaration"].toHashSet() result = success let name = node.getName() if name in known: # Recognized top-level nodes - let - err = node.anyChildInTree("ERROR") - if not err.isNil: - # Bail on errors - gState.printDebug(node) - gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + if gState.getNodeError(node): result = Status.error else: # Process nodes @@ -1750,8 +1748,11 @@ proc processNode(gState: State, node: TSNode): Status = gState.addEnum(node) of "declaration": gState.addDecl(node) - of "function_definition": - gState.addDef(node) + elif name == "function_definition": + # Separate since we only need to check function_declarator for errors and + # not the compound_statement which could have errors but does not impact + # wrapper generation + gState.addDef(node) else: # Unknown, will check child nodes result = unknown diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 08ba604..bfbd4be 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -315,6 +315,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = p = startProcess(getCompiler(), args = args, options = {poStdErrToStdOut, poUsePath}) outp = p.outputStream() line = "" + newHeaders: HashSet[string] # Include content only from file gState.code = "" @@ -323,17 +324,19 @@ proc getPreprocessor*(gState: State, fullpath: string) = # We want to keep blank lines here for comment processing if line.len > 1 and line[0] == '#' and line[1] == ' ': start = false - line = line.sanitizePath(noQuote = true) - if sfile in line or - (DirSep notin line and sfileName in line): + line = line.split('"')[1].sanitizePath(noQuote = true) + if sfile == line or + (DirSep notin line and sfileName == line): start = true elif gState.recurse: - if pDir.Bl or pDir in line: + if (pDir.Bl or pDir in line) and line notin gState.headersProcessed: start = true + newHeaders.incl line else: for inc in includeDirs: - if inc in line: + if line.startsWith(inc) and line notin gState.headersProcessed: start = true + newHeaders.incl line break else: if start: @@ -342,6 +345,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = gState.code.add line & "\n" elif not p.running(): break p.close() + gState.headersProcessed.incl newHeaders # Plugin related @@ -393,3 +397,4 @@ proc expandSymlinkAbs*(path: string): string = result = path.expandFilename().normalizedPath() except: result = path + result = result.sanitizePath(noQuote = true) \ No newline at end of file diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 3b7c2cc..bbe029f 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -399,3 +399,12 @@ proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if name != "comment": result.add(name) + +proc getNodeError*(gState: State, node: TSNode): bool = + let + err = node.anyChildInTree("ERROR") + if not err.isNil: + # Bail on errors + gState.printDebug(node) + gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + result = true \ No newline at end of file From f191ea7244df545c3dc7b4dc2451a600694fe298 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 23:55:26 -0500 Subject: [PATCH 210/255] Fix #237 - anonymous nested struct/unions, fix #236 - unnamed enums --- CHANGES.md | 8 +++- README.md | 5 ++- nimterop/toastlib/ast2.nim | 78 +++++++++++++++++++++++++++++++++----- tests/include/tast2.h | 41 ++++++++++++++++++++ tests/tast2.nim | 20 +++++++++- tests/tnimterop_c.nim | 2 +- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 43f1d34..e85d261 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. +- Nameless enum values are no longer typed to the made-up enum type name, they are instead typed as `cint` to match the underlying type. This allows using such enums without having to depend on the made-up name which could change if enum ordering changes upstream. [#236][i236] (since v0.6.1) + ### New functionality - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. @@ -35,6 +37,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - `cImport()` can now write the generated wrapper output to a user-defined file with the `nimFile` param. [#127][i127] (since v0.6.1) +- Nimterop now supports anonymous nested structs/unions but it only works correctly for unions when `noHeader` is turned off (the default). This is because Nim does not support nested structs/unions and is unaware of the underlying memory structure. [#237][i237] (since v0.6.1) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -128,4 +132,6 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 [i181]: https://github.com/nimterop/nimterop/issues/181 [i196]: https://github.com/nimterop/nimterop/issues/196 [i197]: https://github.com/nimterop/nimterop/issues/197 -[i200]: https://github.com/nimterop/nimterop/issues/200 \ No newline at end of file +[i200]: https://github.com/nimterop/nimterop/issues/200 +[i236]: https://github.com/nimterop/nimterop/issues/236 +[i237]: https://github.com/nimterop/nimterop/issues/237 \ No newline at end of file diff --git a/README.md b/README.md index b054d97..f9e9307 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,9 @@ This will download and install nimterop in the standard Nimble package location, ## Usage -Nimterop can be used in two ways: +Nimterop can be used in three ways: - Creating a wrapper file - a `.nim` file that contains calls to the high-level API that can download and build the C library as well as generate the required Nim code to interface with the library. This wrapper file can then be imported into Nim code like any other module and it will be processed at compile time. +- Same as the first option except using the `nimFile` param to `cImport()` to write the generated wrapper to a file during build time just once and then importing that generated wrapper into the application like any other Nim module. - Using the command line `toast` tool to generate the Nim code which can then be stored into a file and imported separately. Any combination of the above is possible - only download, build or wrapping and nimterop avoids imposing any particular workflow. @@ -169,7 +170,7 @@ For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol a For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. -While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. The user will need to choose based on the library in question. +While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. Further, anonymous nested structs/unions within unions will be rendered incorrectly by Nim since it is unaware of the true memory structure of the type. The user will need to choose based on the library in question. Going further, the `{.dynlib: "path/to/libheader.so".}` pragma can be used to inform Nim to load the library at runtime and link the function instead of linking at compile time. This enables creation of a wrapper that does not need the library present at compile time. diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 43ec8d3..fb2b55f 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -704,6 +704,17 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = edecl = node[i].anyChildInTree("enumerator_list") commentNodes = gState.getCommentNodes(node[i]) + # Check if struct/union field is anonymous + isNamedField = block: + var found = false + if not fdecl.isNil: + var sibling = fdecl.tsNodeParent().tsNodeNextNamedSibling() + while not sibling.isNil and sibling.getName() != "field_identifier": + sibling = sibling.tsNodeNextNamedSibling() + if not sibling.isNil: + found = true + found + # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` (processed, tname) = @@ -725,11 +736,48 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = if processed != success: return nil - # Add nkIdentDefs for each field - for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): - if not field.isNil: - field.comment = gState.getCommentsStr(commentNodes) - result.add field + if not fdecl.isNil and not isNamedField: + # Since anonymous, add fields directly to this struct/union + + # nkTypeDef( <= last + # nkPragmaExpr( + # .. + # ), + # nkEmpty(), + # nkObjectTy( <= last[2] + # nkEmpty(), + # nkEmpty(), + # nkRecList( <= last[2][2] + # nkIdentDefs( <= field1 + # .. + # ), + # nkIdentDefs( <= field2 + # .. + # ) + # ) + # ) + # ) + let + last = gState.typeSection[^1] + obj = + if last.len > 2 and last[2].kind == nkObjectTy: + last[2] + else: nil + recList = + if not obj.isNil and obj.len > 2 and obj[2].kind == nkRecList: + obj[2] + else: nil + + if not recList.isNil: + for identdef in recList: + result.add identdef + gState.typeSection.sons.del(gState.typeSection.len-1) + else: + # Add nkIdentDefs for each field + for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): + if not field.isNil: + field.comment = gState.getCommentsStr(commentNodes) + result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object @@ -1444,14 +1492,16 @@ proc addEnum(gState: State, node: TSNode) = fval = "" if prev.Bl: # Starting default value - fval = &"(0).{name}" + fval = &"(0)" else: # One greater than previous - fval = &"({prev} + 1).{name}" + fval = &"({prev} + 1)" if en.len > 1 and en[1].getName() in gEnumVals: + # Enum value specified, evaluate later, don't use calculated value fieldDeclarations.add((fname, forigname, "", gState.getNodeVal(en[1]), commentNodes)) else: + # Set calculated value fieldDeclarations.add((fname, forigname, fval, "", commentNodes)) fnames.incl fname @@ -1464,10 +1514,18 @@ proc addEnum(gState: State, node: TSNode) = # parseCExpression requires all const identifiers to be present for the enum for (fname, forigname, fval, cexpr, commentNodes) in fieldDeclarations: let - fval = + fval = block: + var fval = fval if fval.Bl: - "(" & $gState.parseCExpression(cexpr, name) & ")." & name - else: fval + # Evaluate enum value from expression + fval = &"({$gState.parseCExpression(cexpr, name)})" + if origname.nBl: + # Named enum so cast to type - #236 + fval &= &".{name}" + else: + # Cast to cint to match underlying type + fval &= ".cint" + fval # Cannot use newConstDef() since parseString(fval) adds backticks to and/or constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 89ad486..06a0078 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -271,6 +271,27 @@ struct TestMyInt { MyInt f1; }; +// Issue #237 +typedef union sx_ivec3 { + struct { + int x; + int y; + struct z { + int z; + }; + }; + + int n[3]; +} sx_ivec3; + +// Issue #236 +enum { + SG_INVALID_ID = 0, + SG_NUM_SHADER_STAGES = 2, + SG_MAX_MIPMAPS = 16, + SG_MAX_TEXTUREARRAY_LAYERS = 128 +}; + // DUPLICATES @@ -546,6 +567,26 @@ struct TestMyInt { MyInt f1; }; +// Issue #237 +typedef union sx_ivec3 { + struct { + int x; + int y; + struct z { + int z; + }; + }; + + int n[3]; +} sx_ivec3; + +// Issue #236 +enum { + SG_INVALID_ID = 0, + SG_NUM_SHADER_STAGES = 2, + SG_MAX_MIPMAPS = 16, + SG_MAX_TEXTUREARRAY_LAYERS = 128 +}; #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index 669ec98..ade5dff 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -353,6 +353,7 @@ checkPragmas(U2, pHeaderBy & @["union"], istype = false) var u2: U2 u2.f1 = addr a15.a2[0] +assert PANEL_WINDOW is nk_panel_type assert PANEL_WINDOW == 1 assert PANEL_GROUP == 2 assert PANEL_POPUP == 4 @@ -497,4 +498,21 @@ when not defined(NOHEADER): when declared(MyInt): assert false, "MyInt is defined!" testFields(TestMyInt, "f1!cint") -checkPragmas(TestMyInt, pHeaderBy, isType = false) \ No newline at end of file +checkPragmas(TestMyInt, pHeaderBy, isType = false) + +# #237 +assert sx_ivec3 is object +testFields(sx_ivec3, "x|y|z|n!cint|cint|cint|array[3, cint]") +checkPragmas(sx_ivec3, pHeaderBy & @["union"], istype = false) +var sx: sx_ivec3 +sx.x = 5 +assert sx.n[0] == 5 +when not defined(NOHEADER): + # Nim doesn't know of the anonymous nested struct so when the header + # isn't present, the test below breaks + sx.n[1] = 4 + assert sx.y == 4 + +# #236 +assert SG_MAX_MIPMAPS is cint +assert SG_MAX_MIPMAPS == 16 \ No newline at end of file diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index ef814ea..ac7193c 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -70,7 +70,7 @@ var e: ENUM e2: ENUM2 = enum5 - e3: Enum_testh1 = enum7 + e3 = enum7 e4: ENUM4 = enum11 vptr: VOIDPTR From 1ad32e4574a70d48cc0f50e71f354207bcdd2bd0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 01:46:27 -0500 Subject: [PATCH 211/255] Fix missing preprocessor errors, toast errors, cDisableCache not working, print generated wrapper location --- nimterop/cimport.nim | 24 ++++++++---------------- nimterop/toastlib/getters.nim | 7 +++++++ tests/include/tast2.h | 2 +- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index d992c04..114ab80 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -82,18 +82,6 @@ proc getCacheValue(fullpaths: seq[string]): string = for fullpath in fullpaths: result &= getCacheValue(fullpath) -proc getToastError(output: string): string = - # Filter out preprocessor errors - for line in output.splitLines(): - if "fatal error:" in line.toLowerAscii: - if result.len == 0: - result = "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" - result &= "\n\nERROR:$1\n" % line.split("fatal error:")[1] - - # Toast error - if result.Bl: - result = "\n\n" & output - proc getNimCheckError(nimFile: string) = let (check, _) = execAction( @@ -159,7 +147,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" when defined(Windows): result = result.replace(DirSep, '/') - if not fileExists(result) or compileOption("forceBuild"): + if not fileExists(result) or gStateCT.nocache or compileOption("forceBuild"): let dir = result.parentDir() if not dirExists(dir): @@ -168,8 +156,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" cmd.add &" -o {result.sanitizePath}" var - (_, ret) = execAction(cmd, die = false) - doAssert ret == 0, getToastError(result.readFile()) + (output, ret) = execAction(cmd, die = false) + doAssert ret == 0, result.readFile() & output macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not @@ -577,6 +565,8 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl if gStateCT.debug: gecho nimFile.readFile() + gecho "# Saved to " & nimFile + try: let nimFileNode = newStrLitNode(nimFile.changeFileExt("")) @@ -673,7 +663,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() - if not fileExists(nimFile) or compileOption("forceBuild"): + if not fileExists(nimFile) or gStateCT.nocache or compileOption("forceBuild"): var cmd = when defined(Windows): "cmd /c " else: "" cmd &= &"c2nim {hFile} --header:{header} --out:{nimFile.sanitizePath}" @@ -699,6 +689,8 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if gStateCT.debug: gecho nimFile.readFile() + gecho "# Saved to " & nimFile + try: let nimFileNode = newStrLitNode(nimFile.changeFileExt("")) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index bfbd4be..df930bb 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -338,6 +338,10 @@ proc getPreprocessor*(gState: State, fullpath: string) = start = true newHeaders.incl line break + elif ": fatal error:" in line: + doAssert false, + "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" & + "\n\nERROR:$1\n" % line.split(": fatal error:")[1] else: if start: if "#undef" in line: @@ -345,6 +349,9 @@ proc getPreprocessor*(gState: State, fullpath: string) = gState.code.add line & "\n" elif not p.running(): break p.close() + assert p.peekExitCode() == 0, + gState.code & "\n\nFailed in preprocessing:\n " & + getCompiler() & " " & args.join(" ") gState.headersProcessed.incl newHeaders # Plugin related diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 06a0078..f9e0017 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -276,7 +276,7 @@ typedef union sx_ivec3 { struct { int x; int y; - struct z { + struct { int z; }; }; From 4d344e5807c17d8644cf23753d8445443cdb72bb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 16:17:34 -0500 Subject: [PATCH 212/255] Handle (*field), ident starting w/ digit error, fix detection of plugin and toast errors --- nimterop/cimport.nim | 2 +- nimterop/toastlib/ast2.nim | 3 ++- nimterop/toastlib/getters.nim | 13 ++++++++----- nimterop/toastlib/tshelp.nim | 2 +- tests/include/tast2.h | 7 +++++++ tests/tast2.nim | 6 +++++- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 114ab80..cedd903 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -157,7 +157,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" var (output, ret) = execAction(cmd, die = false) - doAssert ret == 0, result.readFile() & output + doAssert ret == 0, "\n\n" & (if result.fileExists(): result.readFile() else: "") & output macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index fb2b55f..083d4b2 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -523,7 +523,8 @@ proc newIdentDef(gState: State, name: string, node: TSNode, tname: string, tinfo result.add pident let - count = node[offset].getPtrCount() + # Could be parenthesized + count = node[offset].getAtom().tsNodeParent().getPtrCount(reverse = true) if count > 0: result.add gState.newPtrTree(count, tident) else: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index df930bb..1e70d59 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -167,13 +167,15 @@ proc checkIdentifier(name, kind, parent, origName: string) = if name.nBl: let - origStr = if name != origName: &", originally '{origName}' before 'cPlugin:onSymbol()', still" else: "" - errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} contains $1 " & + origStr = if name != origName: &", originally '{origName}' before 'cPlugin:onSymbol()'," else: "" + errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} $1 " & "which Nim does not allow. Use toast flag '$2' or 'cPlugin()' to modify." - doAssert name[0] != '_' and name[^1] != '_', errmsg % ["leading/trailing underscores '_'", "--prefix or --suffix"] + doAssert name[0] != '_' and name[^1] != '_', errmsg % ["has leading/trailing underscores '_'", "--prefix or --suffix"] - doAssert (not name.contains("__")): errmsg % ["consecutive underscores '_'", "--replace"] + doAssert (not name.contains("__")): errmsg % ["has consecutive underscores '_'", "--replace"] + + doAssert not name[0].isDigit(), errmsg % [&"starts with a digit '{name[0]}'", "--prefix"] # Cannot blank out symbols which are fields or params # @@ -381,7 +383,8 @@ proc loadPlugin*(gState: State, sourcePath: string) = # Compile plugin as library with `markAndSweep` GC cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" - discard execAction(cmd) + (output, ret) = execAction(cmd, die = false) + doAssert ret == 0, output & "\nFailed to compile cPlugin()\n\ncmd: " & cmd doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath let lib = loadLib(pdll) diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index bbe029f..81c8d0d 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -116,7 +116,7 @@ proc getXCount*(node: TSNode, ntype: string, reverse = false): int = break proc getPtrCount*(node: TSNode, reverse = false): int = - node.getXCount("pointer_declarator") + node.getXCount("pointer_declarator", reverse) proc getArrayCount*(node: TSNode, reverse = false): int = node.getXCount("array_declarator") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index f9e0017..38c5f18 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -292,6 +292,9 @@ enum { SG_MAX_TEXTUREARRAY_LAYERS = 128 }; +struct parenpoin { + void (*gtk_reserved1); +}; // DUPLICATES @@ -588,6 +591,10 @@ enum { SG_MAX_TEXTUREARRAY_LAYERS = 128 }; +struct parenpoin { + void (*__gtk_reserved1); +}; + #endif #ifdef __cplusplus diff --git a/tests/tast2.nim b/tests/tast2.nim index ade5dff..17edfb0 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -515,4 +515,8 @@ when not defined(NOHEADER): # #236 assert SG_MAX_MIPMAPS is cint -assert SG_MAX_MIPMAPS == 16 \ No newline at end of file +assert SG_MAX_MIPMAPS == 16 + +assert parenpoin is object +var pp: parenpoin +assert pp.gtk_reserved1 is pointer \ No newline at end of file From 03d1b54d6c10317a7bd005f35ecb7a9b426051fb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 23:51:52 -0500 Subject: [PATCH 213/255] gitPull non-master default branch, rmFile fix --- nimterop/build/shell.nim | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 2b29734..f07f70c 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -155,7 +155,7 @@ proc rmFile*(source: string, dir = false) = if dir: "rd /s/q" else: - "del /s/q/f" + "del /q/f" else: "rm -rf" exists = @@ -335,6 +335,18 @@ proc gitAtCheckout*(outdir, checkout: string): bool = result = checkout in execAction( &"cd {outdir.sanitizePath} && git log --decorate --no-color -n 1 --format=oneline").output +proc gitDefaultBranch*(outdir: string): string = + ## Get the default branch for a git repository before it is pulled + result = "master" + let + output = execAction( + &"cd {outdir.sanitizePath} && git remote show origin" + ).output + + for line in output.splitLines(): + if "HEAD branch: " in line: + result = line.split("branch: ")[1].strip() + proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## @@ -375,15 +387,14 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false # In case directory has old files from another run discard execAction(&"cd {outdirQ} && git clean -fxd") - if checkout.len != 0: - if not quiet: - gecho "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 3) - discard execAction(&"cd {outdirQ} && git checkout {checkout}") - else: - if not quiet: - gecho "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) + # Checkout specified branch/tag/commit or default branch - typically master + let + checkout = if checkout.Bl: gitDefaultBranch(outdir) else: checkout + + if not quiet: + gecho "# Checking out " & checkout + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) + discard execAction(&"cd {outdirQ} && git checkout {checkout}") proc gitTags*(outdir: string): seq[string] = ## Get all the git tags in the specified directory From 90c5f54dc3b21c32e6666065c7a192c89a37fb4f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 4 Jul 2020 00:49:46 -0500 Subject: [PATCH 214/255] v0.6.2 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 628ef40..adbd171 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.1" +version = "0.6.2" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 37cdc5b94ec42c268fc2f7dc63cea967b3d521cd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 11 Jul 2020 17:29:21 -0500 Subject: [PATCH 215/255] Add jbbFlags to customize xxxJBB --- CHANGES.md | 4 ++- nimterop/build/getheader.nim | 42 ++++++++++++++++++++++--------- nimterop/build/jbb.nim | 48 ++++++++++++++++++++++++------------ 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e85d261..70e1f76 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 ### Breaking changes @@ -39,6 +39,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - Nimterop now supports anonymous nested structs/unions but it only works correctly for unions when `noHeader` is turned off (the default). This is because Nim does not support nested structs/unions and is unaware of the underlying memory structure. [#237][i237] (since v0.6.1) +- `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 6972ccc..d5b398a 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -141,18 +141,14 @@ proc getConanLDeps(outdir: string): seq[string] = result = pkg.getConanLDeps(outdir) -proc getJBBPath(header, uri, outdir, version: string): string = +proc getJBBPath(header, uri, flags, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] hasVersion = version.len != 0 var - ver = - if spl.len == 2: - spl[1] - else: - "" + ver = if spl.len == 2: spl[1] else: "" if ver.len != 0: if "$#" in ver or "$1" in ver: @@ -166,6 +162,20 @@ proc getJBBPath(header, uri, outdir, version: string): string = let pkg = newJBBPackage(name, ver) + + # Handle `jbbFlags` + if flags.nBl: + if flags.startsWith("giturl="): + let + val = flags["giturl=".len .. ^1] + if val.contains("://"): + pkg.baseUrl = val + else: + pkg.baseUrl = "https://github.com/" & val + elif flags.startsWith("url="): + pkg.baseUrl = flags["url=".len .. ^1] + pkg.isGit = false + downloadJBB(pkg, outdir) result = findFile(header, outdir) @@ -217,7 +227,8 @@ macro getHeader*( conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = + jbbFlags: static[string] = "", altNames: static[string] = "", + buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. @@ -289,6 +300,13 @@ macro getHeader*( ## `cmake` and `make` in case additional configuration is required as part of the build ## process. ## + ## `jbbFlags` allows changing the BinaryBuilder.org defaults: + ## - `giturl=customUrl` changes the default `https://github.com/JuliaBinaryWrappers` to + ## another Git URL. If no hostname is specified, `https://github.com` is assumed. + ## - `url=customUrl` uses regular HTTP instead of Git and looks for `Artifacts.toml` and + ## `Project.toml` files at that location. `$1` or `$#` are replaced with the version + ## if specified. + ## ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma @@ -384,7 +402,7 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, jbburi, jbbFlags, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: @@ -392,7 +410,7 @@ macro getHeader*( elif `nameConan`: getConanPath(header, conanuri, outdir, version, shared) elif `nameJBB`: - getJBBPath(header, jbburi, outdir, version) + getJBBPath(header, jbburi, jbbFlags, outdir, version) else: getLocalPath(header, outdir) @@ -423,7 +441,8 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): @@ -458,7 +477,8 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + `outdir`, `version`, not `nameStatic`) static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 0fb163f..b9272c7 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -12,7 +12,10 @@ type name*: string version*: string - url*: string + baseUrl*: string # Location to find package + isGit*: bool # Git or HTTP + + url*: string # Download URL sharedLibs*: seq[string] staticLibs*: seq[string] @@ -20,7 +23,7 @@ type const # JBB URLs - jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl" + jbbBaseUrl = "https://github.com/JuliaBinaryWrappers" jbbInfo = "jbbinfo.json" jbbProject = "Project.toml" @@ -45,6 +48,8 @@ proc newJBBPackage*(name, version: string): JBBPackage = result = new(JBBPackage) result.name = name result.version = version + result.baseUrl = jbbBaseUrl + result.isGit = true proc parseJBBProject(pkg: JBBPackage, outdir: string) = # Get all dependencies from Project.toml @@ -127,21 +132,32 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = let path = outdir / "repos" / pkg.name - gitPull( - jbbBaseUrl % pkg.name, - outdir = path, - plist = "*.toml", - "master", - quiet = true - ) + if pkg.isGit: + # Get package info using Git + gitPull( + pkg.baseUrl & ("/$1_jll.jl" % pkg.name), + outdir = path, + plist = "*.toml", + "master", + quiet = true + ) - if pkg.version.len != 0: - # Checkout correct tag - let - tags = gitTags(path) - for i in tags.len - 1 .. 0: - if pkg.version in tags[i] and i != tags.len - 1: - gitCheckout(path, tags[i-1]) + if pkg.version.len != 0: + # Checkout correct tag + let + tags = gitTags(path) + for i in tags.len - 1 .. 0: + if pkg.version in tags[i] and i != tags.len - 1: + gitCheckout(path, tags[i-1]) + else: + # Download package info from HTTP + var + url = pkg.baseUrl + if "$#" in url or "$1" in url: + doAssert pkg.version.len != 0, "Need version for custom BinaryBuilder.org url: " & url + url = url % pkg.version + downloadUrl(url & "Artifacts.toml", path, quiet = true) + downloadUrl(url & "Project.toml", path, quiet = true) pkg.parseJBBProject(path) pkg.parseJBBArtifacts(path) From fb2acc752a283223b5c4647bfe85d75cd23a5b4f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 00:12:41 -0500 Subject: [PATCH 216/255] v0.6.3 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index adbd171..16a6c6a 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.2" +version = "0.6.3" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From a9887cc6b2569bddd1daf56ae66960fb500be3e9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 17:35:29 -0500 Subject: [PATCH 217/255] Add ability to exclude files or directories from wrapped output --- CHANGES.md | 4 +++ README.md | 2 ++ nimterop/cimport.nim | 56 ++++++++++++++++++++++++++++------- nimterop/globals.nim | 1 + nimterop/toast.nim | 6 +++- nimterop/toastlib/getters.nim | 16 ++++++++-- 6 files changed, 70 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 70e1f76..23aa2bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,12 +41,16 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) +- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) - `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) +- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. + ## Version 0.5.0 diff --git a/README.md b/README.md index f9e9307..0e79a6f 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ cDefine("HAS_ABC") # Set #defines for preprocessor and compiler cDefine("HAS_ABC", "DEF") cIncludeDir("clib/include") # Setup any include directories +cExclude("clib/file.h") # Exclude file from wrapped output cImport("clib.h") # Generate wrappers for header specified @@ -219,6 +220,7 @@ Options: -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path + -X=, --exclude= strings {} files or directories to exclude from the wrapped output -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index cedd903..95004ad 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -120,6 +120,9 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.includeDirs: cmd.add &" --includeDirs+={i.sanitizePath}" + for i in gStateCT.exclude: + cmd.add &" --exclude+={i.sanitizePath}" + if not noNimout: cmd.add &" --pnim" @@ -415,21 +418,52 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = if dir notin gStateCT.searchDirs: gStateCT.searchDirs.add(dir) -macro cIncludeDir*(dir: static string): untyped = +macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = + ## Add include directories that are forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of these include directories should + ## not be included in the wrapped output. + for dir in dirs: + var dir = interpPath(dir) + result = newNimNode(nnkStmtList) + + let fullpath = findPath(dir) + if fullpath notin gStateCT.includeDirs: + gStateCT.includeDirs.add(fullpath) + if exclude: + gStateCT.exclude.add(fullpath) + let str = &"-I{fullpath.quoteShell}" + result.add quote do: + {.passC: `str`.} + if gStateCT.debug: + gecho result.repr + +macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - var dir = interpPath(dir) - result = newNimNode(nnkStmtList) + ## + ## Set `exclude = true` if the contents of this include directory should + ## not be included in the wrapped output. + return quote do: + cIncludeDir(@[`dir`], `exclude` == 1) - let fullpath = findPath(dir) - if fullpath notin gStateCT.includeDirs: - gStateCT.includeDirs.add(fullpath) - let str = &"-I{fullpath.quoteShell}" - result.add quote do: - {.passC: `str`.} - if gStateCT.debug: - gecho result.repr +macro cExclude*(paths: static seq[string]): untyped = + ## Exclude specified paths - files or directories from the wrapped output + ## + ## Full path to file or directory is required. + result = newNimNode(nnkStmtList) + for path in paths: + gStateCT.exclude.add path + +macro cExclude*(path: static string): untyped = + ## Exclude specified path - file or directory from the wrapped output. + ## + ## Full path to file or directory is required. + return quote do: + cExclude(@[`path`]) proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 17b9518..89e403f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -19,6 +19,7 @@ type debug*: bool # `cDebug()` or `--debug | -d` to enable debug mode defines*: seq[string] # Symbols added by `cDefine()` and `--define | -D` for C/C++ preprocessor/compiler dynlib*: string # `cImport(dynlib)` or `--dynlib | -l` to specify variable containing library name + exclude*: seq[string] # files or directories to exclude from the wrapped output feature*: seq[Feature] # `--feature | -f` feature flags enabled includeDirs*: seq[string] # Paths added by `cIncludeDir()` and `--includeDirs | -I` for C/C++ preprocessor/compiler mode*: string # `cImport(mode)` or `--mode | -m` to override detected compiler mode - c or cpp diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 1cda239..20dc79e 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -13,7 +13,7 @@ var preMainOut = "" proc process(gState: State, path: string) = - doAssert existsFile(path), &"Invalid path {path}" + doAssert fileExists(path), &"Invalid path {path}" if gState.mode.Bl: gState.mode = getCompilerMode(path) @@ -38,6 +38,7 @@ proc main( debug = false, defines: seq[string] = @[], dynlib: string = "", + exclude: seq[string] = @[], feature: seq[Feature] = @[], includeDirs: seq[string] = @[], mode = "", @@ -65,6 +66,7 @@ proc main( debug: debug, defines: defines, dynlib: dynlib, + exclude: exclude, feature: feature, includeDirs: includeDirs, mode: mode, @@ -222,6 +224,7 @@ when isMainModule: "debug": "enable debug output", "defines": "definitions to pass to preprocessor", "dynlib": "{.dynlib.} pragma to import symbols - Nim const string or file path", + "exclude": "files or directories to exclude from the wrapped output", "feature": "flags to enable experimental features", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", @@ -247,6 +250,7 @@ when isMainModule: "debug": 'd', "defines": 'D', "dynlib": 'l', + "exclude": 'X', "feature": 'f', "includeDirs": 'I', "noComments": 'c', diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 1e70d59..8c6abe1 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -286,7 +286,16 @@ proc getKeyword*(kind: NimSymKind): string = proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) +proc isIncluded(gState: State, file: string): bool {.inline.} = + # Check if the specified file should be excluded from wrapped output + if gState.exclude.nBl: + for excl in gState.exclude: + if file.startsWith(excl): + return + result = true + proc getPreprocessor*(gState: State, fullpath: string) = + # Get preprocessed output from the C/C++ compiler var args: seq[string] start = false @@ -332,14 +341,15 @@ proc getPreprocessor*(gState: State, fullpath: string) = start = true elif gState.recurse: if (pDir.Bl or pDir in line) and line notin gState.headersProcessed: - start = true newHeaders.incl line + start = gState.isIncluded(line) else: for inc in includeDirs: if line.startsWith(inc) and line notin gState.headersProcessed: - start = true newHeaders.incl line - break + start = gState.isIncluded(line) + if start: + break elif ": fatal error:" in line: doAssert false, "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" & From d7c94bb70bb754ffbe85669e550589089d6597d1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 22:21:35 -0500 Subject: [PATCH 218/255] Bump to v0.6.4 --- .travis.yml | 3 ++- CHANGES.md | 4 ++-- nimterop.nimble | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 173da68..b58b29d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 - - BRANCH=1.2.2 + - BRANCH=1.2.4 - BRANCH=devel cache: @@ -26,6 +26,7 @@ install: - source travis.sh script: + - set -e - nimble develop -y - nimble test - nimble --verbose --nimbleDir:`pwd`/build/fakenimble install nimterop@#head -y diff --git a/CHANGES.md b/CHANGES.md index 23aa2bb..91cecac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,7 +41,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) -- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. +- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. (since v0.6.4) ### Other improvements @@ -49,7 +49,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) -- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. +- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. (since v0.6.4) ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index 16a6c6a..1eb35ef 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.3" +version = "0.6.4" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 27fcecaa26a186dbc1e45c475baefe7182abb425 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 13 Jul 2020 17:47:17 -0500 Subject: [PATCH 219/255] Caching fixes, wchar_t --- nimterop/cimport.nim | 42 ++++++++++++++++++++++++++++++----- nimterop/toastlib/getters.nim | 4 ++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 95004ad..aa7895a 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -142,15 +142,35 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - result = if outFile.nBl: fixRelFile(outFile) else: - # Generate filename for toast output if not specified - getNimteropCacheDir() / "toastCache" / "nimterop_" & + let + cacheFile = getNimteropCacheDir() / "toastCache" / "nimterop_" & ($(cmd & cacheKey).hash().abs()).addFileExt(ext) + if outFile.nBl: + result = fixRelFile(outFile) + else: + result = cacheFile + when defined(Windows): result = result.replace(DirSep, '/') - if not fileExists(result) or gStateCT.nocache or compileOption("forceBuild"): + let + # When to regenerate the wrapper + regen = + if gStateCT.nocache or compileOption("forceBuild"): + # No caching or forced + true + elif not fileExists(result): + # Cache or outfile doesn't exist + true + elif outFile.nBl and (not fileExists(cacheFile) or + result.getFileDate() > cacheFile.getFileDate()): + # Outfile exists but cache doesn't or outdated + true + else: + false + + if regen: let dir = result.parentDir() if not dirExists(dir): @@ -160,7 +180,19 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" var (output, ret) = execAction(cmd, die = false) - doAssert ret == 0, "\n\n" & (if result.fileExists(): result.readFile() else: "") & output + if ret != 0: + # If toast fails, print failure to output and delete any generated files + let errout = if result.fileExists(): result.readFile() & output else: output + rmFile(result) + doAssert false, "\n\n" & errout & "\n" + + # Write empty cache file to track changes when outFile specified + if outFile.nBl: + let dir = cacheFile.parentDir() + if not dirExists(dir): + mkdir(dir) + + writeFile(cacheFile, "") macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 8c6abe1..8a4586d 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -139,7 +139,7 @@ when defined(cpp): # not defined in nor any other header). type wchar_t* {.importc.} = object else: - type wchar_t* {.importc, header:"".} = object + type wchar_t* {.importc, header:"stddef.h".} = object """, "va_list": """ @@ -417,4 +417,4 @@ proc expandSymlinkAbs*(path: string): string = result = path.expandFilename().normalizedPath() except: result = path - result = result.sanitizePath(noQuote = true) \ No newline at end of file + result = result.sanitizePath(noQuote = true) From 6ffacf92fc19d2cd0754e2409868a2dca4501e93 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Jul 2020 18:08:59 -0500 Subject: [PATCH 220/255] Forward pragmas to toast --- README.md | 2 + nimterop/build/conan.nim | 20 ++-- nimterop/build/jbb.nim | 17 ++- nimterop/build/nimconf.nim | 4 + nimterop/cimport.nim | 212 ++++++++++++++++--------------------- nimterop/globals.nim | 5 +- nimterop/template.nim | 111 ------------------- nimterop/templite.nim | 36 ------- nimterop/toast.nim | 9 ++ nimterop/toastlib/ast2.nim | 24 ++++- tests/libssh2.nim | 4 +- tests/rsa.nim | 4 +- tests/tast2.nim | 2 +- tests/tsoloud.nim | 8 +- tests/zlib.nim | 10 +- 15 files changed, 162 insertions(+), 306 deletions(-) delete mode 100644 nimterop/template.nim delete mode 100644 nimterop/templite.nim diff --git a/README.md b/README.md index 0e79a6f..cda3b5e 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ cImport("clib.h") # Generate wrappers for header specified cCompile("clib/src/*.c") # Compile in any implementation source files ``` +All `{.compileTime.}` procs must be used in a compile time context, like `cDebug()` and `cDisableCaching()` above. + Module documentation for the wrapper API can be found [here](https://nimterop.github.io/nimterop/cimport.html). #### Preprocessing diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 792de01..dd43dc9 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -79,10 +79,6 @@ proc jsonGet(url: string): JsonNode = discard rmFile(file) -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - proc `==`*(pkg1, pkg2: ConanPackage): bool = ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -282,9 +278,8 @@ proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = proc loadConanInfo*(outdir: string): ConanPackage = ## Load cached package info from `outdir/conaninfo.json` - fixOutDir() let - file = outdir / conanInfo + file = fixRelPath(outdir) / conanInfo if fileExists(file): when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -297,9 +292,8 @@ proc loadConanInfo*(outdir: string): ConanPackage = proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` - fixOutDir() let - file = outdir / conanInfo + file = fixRelPath(outdir) / conanInfo when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): writeFile(file, $$pkg) @@ -329,11 +323,11 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## Download specific `revision` of `bld` to `outdir` ## ## If omitted, the latest revision (first) is downloaded - fixOutDir() - doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() let + outdir = fixRelPath(outdir) + revision = if revision.len != 0: revision @@ -376,8 +370,9 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = ## ## High-level API that handles the end to end Conan process flow to find ## latest package binary and downloads and extracts it to `outdir`. - fixOutDir() let + outdir = fixRelPath(outdir) + pkg = if pkg.version.len == 0: searchConan(pkg) @@ -415,7 +410,8 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = ## ## This is not required for shared libs since conan builds them ## with all dependencies statically linked in - fixOutDir() + let + outdir = fixRelPath(outdir) if bld.options["shared"] == "False": for req in bld.requires: let diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index b9272c7..a430a55 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -33,10 +33,6 @@ var # Reuse dependencies already downloaded gJBBRequires {.compileTime.}: Table[string, JBBPackage] -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -164,9 +160,8 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = proc loadJBBInfo*(outdir: string): JBBPackage = ## Load cached package info from `outdir/jbbinfo.json` - fixOutDir() let - file = outdir / jbbInfo + file = fixRelPath(outdir) / jbbInfo if fileExists(file): when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -179,9 +174,8 @@ proc loadJBBInfo*(outdir: string): JBBPackage = proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` - fixOutDir() let - file = outdir / jbbInfo + file = fixRelPath(outdir) / jbbInfo when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): writeFile(file, $$pkg) @@ -194,7 +188,9 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = ## ## High-level API that handles the end to end JBB process flow to find ## latest package binary and downloads and extracts it to `outdir`. - fixOutDir() + let + outdir = fixRelPath(outdir) + if main: let cpkg = loadJBBInfo(outdir) @@ -226,7 +222,8 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = ## Download all required dependencies of this `pkg` - fixOutDir() + let + outdir = fixRelPath(outdir) for i in 0 ..< pkg.requires.len: let rpkg = pkg.requires[i] diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 3bbe521..7624bf2 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -234,3 +234,7 @@ proc getOutDir*(projectDir = ""): string = proc getNimteropCacheDir*(): string = ## Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" + +proc fixRelPath*(path: string): string = + ## If `path` is relative, consider relative to `projectPath` + if path.isAbsolute: path else: getProjectDir() / path \ No newline at end of file diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index aa7895a..6120cb5 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -1,78 +1,17 @@ -##[ -This is the main nimterop import file to help with wrapping C/C++ source code. - -Check out `template.nim `_ -as a starting point for wrapping a new library. The template can be copied and -trimmed down and modified as required. `templite.nim `_ is a shorter -version for more experienced users. - -All `{.compileTime.}` procs must be used in a compile time context, e.g. using: - -.. code-block:: c - - static: - cAddStdDir() - -]## - import hashes, macros, os, strformat, strutils import "."/[globals, paths] import "."/build/[ccompiler, misc, nimconf, shell] -proc interpPath(dir: string): string= - # TODO: more robust: needs a DirSep after "$projpath" - # disabling this interpolation as this is error prone, but other less - # interpolations can be added, eg see https://github.com/nim-lang/Nim/pull/10530 - # result = dir.replace("$projpath", getProjectPath()) - result = dir - -proc joinPathIfRel(path1: string, path2: string): string = - if path2.isAbsolute: - result = path2 - else: - result = joinPath(path1, path2) - proc findPath(path: string, fail = true): string = # Relative to project path - result = joinPathIfRel(getProjectPath(), path).replace("\\", "/") + let + path = fixRelPath(path) + result = path.replace("\\", "/") if not fileExists(result) and not dirExists(result): doAssert (not fail), "File or directory not found: " & path result = "" -proc walkDirImpl(indir, inext: string, file=true): seq[string] = - let - dir = joinPathIfRel(getProjectPath(), indir) - ext = - if inext.nBl: - when not defined(Windows): - "-name " & inext - else: - "\\" & inext - else: - "" - - let - cmd = - when defined(Windows): - if file: - "cmd /c dir /s/b/a-d " & dir.replace("/", "\\") & ext - else: - "cmd /c dir /s/b/ad " & dir.replace("/", "\\") - else: - if file: - "find $1 -type f $2" % [dir, ext] - else: - "find $1 -type d" % dir - - (output, ret) = execAction(cmd, die = false) - - if ret == 0: - result = output.splitLines() - -proc fixRelFile(file: string): string = - if file.isAbsolute(): file else: getProjectDir() / file - proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() @@ -123,6 +62,15 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.exclude: cmd.add &" --exclude+={i.sanitizePath}" + for i in gStateCT.passC: + cmd.add &" --passC+={i.quoteShell}" + + for i in gStateCT.passL: + cmd.add &" --passL+={i.quoteShell}" + + for i in gStateCT.compile: + cmd.add &" --compile+={i.sanitizePath}" + if not noNimout: cmd.add &" --pnim" @@ -147,7 +95,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" ($(cmd & cacheKey).hash().abs()).addFileExt(ext) if outFile.nBl: - result = fixRelFile(outFile) + result = fixRelPath(outFile) else: result = cacheFile @@ -384,7 +332,7 @@ macro cPluginPath*(path: static[string]): untyped = doAssert fileExists(path), "Plugin file not found: " & path cPluginHelper(readFile(path), imports = "") -proc cSearchPath*(path: string): string {.compileTime.}= +proc cSearchPath*(path: string): string {.compileTime.} = ## Get full path to file or directory `path` in search path configured ## using `cAddSearchDir()` and `cAddStdDir()`. ## @@ -416,27 +364,38 @@ proc cDisableCaching*() {.compileTime.} = ## `nim -f` can also be used to flush the cached content. gStateCT.nocache = true -macro cDefine*(name: static string, val: static string = ""): untyped = +macro cDefine*(name: static[string], val: static[string] = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` - result = newNimNode(nnkStmtList) - var str = name - # todo: see https://github.com/nimterop/nimterop/issues/100 for - # edge case of empty strings if val.nBl: str &= &"={val.quoteShell}" if str notin gStateCT.defines: gStateCT.defines.add(str) - str = "-D" & str - result.add quote do: - {.passC: `str`.} +macro cDefine*(values: static seq[string]): untyped = + ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor + ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + for value in values: + let + spl = value.split("=", maxsplit = 1) + name = spl[0] + val = if spl.len == 2: spl[1] else: "" + discard quote do: + cDefine(`name`, `val`) - if gStateCT.debug: - gecho result.repr & "\n" +macro cPassC*(value: static string): untyped = + ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + gStateCT.passC.add value + +macro cPassL*(value: static string): untyped = + ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + gStateCT.passL.add value proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to @@ -445,8 +404,8 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = import nimterop/paths, os static: cAddSearchDir testsIncludeDir() - doAssert cSearchPath("test.h").existsFile - var dir = interpPath(dir) + doAssert cSearchPath("test.h").fileExists + if dir notin gStateCT.searchDirs: gStateCT.searchDirs.add(dir) @@ -458,19 +417,11 @@ macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): unt ## Set `exclude = true` if the contents of these include directories should ## not be included in the wrapped output. for dir in dirs: - var dir = interpPath(dir) - result = newNimNode(nnkStmtList) - let fullpath = findPath(dir) if fullpath notin gStateCT.includeDirs: gStateCT.includeDirs.add(fullpath) if exclude: gStateCT.exclude.add(fullpath) - let str = &"-I{fullpath.quoteShell}" - result.add quote do: - {.passC: `str`.} - if gStateCT.debug: - gecho result.repr macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if @@ -501,16 +452,17 @@ proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search ## path used in calls to `cSearchPath()`. runnableExamples: - static: cAddStdDir() import os - doAssert cSearchPath("math.h").existsFile + static: + cAddStdDir() + doAssert cSearchPath("math.h").fileExists for inc in getGccPaths(mode): cAddSearchDir inc -macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = +macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` ## - ## `path` can be a specific file or contain wildcards: + ## `path` can be a specific file or contain `*` wildcard for filename: ## ## .. code-block:: nim ## @@ -532,26 +484,21 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = ## ## cCompile("path/to/dir", exclude="test2.c") - result = newNimNode(nnkStmtList) - - var - stmt = "" - - proc fcompile(file: string): string = + proc fcompile(file: string) = let (_, fn, ext) = file.splitFile() var ufn = fn uniq = 1 - while ufn in gStateCT.compile: + while ufn in gStateCT.compcache: ufn = fn & $uniq uniq += 1 # - https://github.com/nim-lang/Nim/issues/10299 # - https://github.com/nim-lang/Nim/issues/10486 - gStateCT.compile.add(ufn) + gStateCT.compcache.add(ufn) if fn == ufn: - return "{.compile: \"$#\".}\n" % file.replace("\\", "/") + gStateCT.compile.add file.replace("\\", "/") else: # - https://github.com/nim-lang/Nim/issues/9370 let @@ -559,7 +506,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): cpFile(file, tmpFile) - return "{.compile: \"$#\".}\n" % tmpFile.replace("\\", "/") + gStateCT.compile.add tmpFile.replace("\\", "/") # Due to https://github.com/nim-lang/Nim/issues/9863 # cannot use seq[string] for excludes @@ -572,33 +519,37 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = if excl in file: result = false - proc dcompile(dir, exclude: string, ext=""): string = + proc dcompile(dir, exclude: string, ext="") = let - files = walkDirImpl(dir, ext) + (dir, pat) = + if "*" in dir: + dir.splitPath() + else: + (dir, "") - for f in files: - if f.nBl and f.notExcluded(exclude): - result &= fcompile(f) + for file in walkDirRec(dir): + if ext.nBl or pat.nBl: + let + fext = file.splitFile().ext + if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): + continue + if file.notExcluded(exclude): + fcompile(file) - if path.contains("*") or path.contains("?"): - stmt &= dcompile(path, exclude.strVal()) + if "*" in path: + dcompile(path, exclude) else: let fpath = findPath(path) - if fileExists(fpath) and fpath.notExcluded(exclude.strVal()): - stmt &= fcompile(fpath) + if fileExists(fpath) and fpath.notExcluded(exclude): + fcompile(fpath) elif dirExists(fpath): - if mode.strVal().contains("cpp"): - for i in @["*.cpp", "*.c++", "*.cc", "*.cxx"]: - stmt &= dcompile(fpath, exclude.strVal(), i) + if mode.contains("cpp"): + for i in @[".cpp", ".c++", ".cc", ".cxx"]: + dcompile(fpath, exclude, i) when not defined(Windows): - stmt &= dcompile(fpath, exclude.strVal(), "*.C") + dcompile(fpath, exclude, ".C") else: - stmt &= dcompile(fpath, exclude.strVal(), "*.c") - - result.add stmt.parseStmt() - - if gStateCT.debug: - gecho result.repr + dcompile(fpath, exclude, ".c") macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = @@ -726,7 +677,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") + nimFile = if nimFile.nBl: fixRelPath(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(nimFile) or gStateCT.nocache or compileOption("forceBuild"): @@ -741,8 +692,29 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if flags.nBl: cmd.add &" {flags}" + # Have to create pragmas for c2nim since toast handles this at runtime for i in gStateCT.defines: cmd.add &" --assumedef:{i.quoteShell}" + let str = "-D" & i + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.includeDirs: + let str = &"-I{i.quoteShell}" + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.passC: + result.add quote do: + {.passC: `i`.} + + for i in gStateCT.passL: + result.add quote do: + {.passL: `i`.} + + for i in gStateCT.compile: + result.add quote do: + {.compile: `i`.} let (c2nimout, ret) = execAction(cmd) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 89e403f..2900272 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -15,6 +15,7 @@ type State* = ref object # Command line arguments to toast - some forwarded from cimport.nim + compile*: seq[string] # `--compile` to create `{.compile.}` entries in generated wrapper convention*: string # `--convention | -C` to change calling convention from cdecl default debug*: bool # `cDebug()` or `--debug | -d` to enable debug mode defines*: seq[string] # Symbols added by `cDefine()` and `--define | -D` for C/C++ preprocessor/compiler @@ -26,6 +27,8 @@ type nim*: string # `--nim` to specify full path to Nim compiler noComments*: bool # `--noComments | -c` to disable rendering comments in wrappers noHeader*: bool # `--noHeader | -H` to skip {.header.} pragma in wrapper + passC*: seq[string] # `--passC` to create `{.passC.}` entries in the generated wrapper + passL*: seq[string] # `--passL` to create `{.passL.}` entries in the generated wrapper past*: bool # `--past | -a` to print tree-sitter AST of code pluginSourcePath*: string # `--pluginSourcePath` specified path to plugin file to compile and load pnim*: bool # `--pnim | -n` to render Nim wrapper for header @@ -77,7 +80,7 @@ type wrapperHeader*: string else: # cimport.nim specific - compile*: seq[string] # `cCompile()` list of files already processed + compcache*: seq[string] # `cCompile()` list of files already processed nocache*: bool # `cDisableCaching()` to disable caching of artifacts overrides*: string # `cOverride()` code which gets added to `cPlugin()` output pluginSource*: string # `cPlugin()` generated code to write to plugin file from diff --git a/nimterop/template.nim b/nimterop/template.nim deleted file mode 100644 index c6e846f..0000000 --- a/nimterop/template.nim +++ /dev/null @@ -1,111 +0,0 @@ -import os, strutils - -import nimterop/[cimport, build, paths] - -# Documentation: -# https://github.com/nimterop/nimterop -# https://nimterop.github.io/nimterop/cimport.html - -const - # Location where any sources should get downloaded. Adjust depending on - # actual location of wrapper file relative to project. - baseDir = currentSourcePath.parentDir()/"build" - - # All files and dirs should be inside to baseDir - srcDir = baseDir/"project" - -static: - # Print generated Nim to output - cDebug() - - # Disable caching so that wrapper is generated every time. Useful during - # development. Remove once wrapper is working as expected. - cDisableCaching() - - # Download C/C++ source code from a git repository - gitPull("https://github.com/user/project", outdir = srcDir, plist = """ -include/*.h -src/*.c -""", checkout = "tag/branch/hash") - - # Download source from the web - zip files are auto extracted - downloadUrl("https://hostname.com/file.h", outdir = srcDir) - - # Run GNU configure on the source - when defined(posix): - configure(srcDir, fileThatShouldGetGenerated, flagsToConfigure) - - # Run cmake on the source - cmake(srcDir/"build", fileThatShouldGetGenerated, flagsToCmake) - - # Run standard file/directory operations with mkDir(), cpFile(), mvFile() - - # Edit file contents if required with readFile(), writeFile() and standard - # string operations - - # Run any other external commands with execAction() - - # Skip any symbols from being wrapped - cSkipSymbol(@["type1", "proc2"]) - -# Manually wrap any symbols since nimterop cannot or incorrectly wraps them -cOverride: - # Standard Nim code to wrap types, consts, procs, etc. - type - symbol = object - -# Specify include directories for gcc and Nim -cIncludeDir(srcDir/"include") - -# Define global symbols -cDefine("SYMBOL", "value") - -# Any global compiler options -{.passC: "flags".} - -# Any global linker options -{.passL: "flags".} - -# Compile in any common source code -cCompile(srcDir/"file.c") - -# Perform OS specific tasks -when defined(Windows): - # Windows specific symbols, options and files - - # Dynamic library to link against - const dynlibFile = - when defined(cpu64): - "xyz64.dll" - else: - "xyz32.dll" -elif defined(posix): - # Common posix symbols, options and files - - when defined(linux): - # Linux specific - const dynlibFile = "libxyz.so(.2|.1|)" - elif defined(osx): - # MacOSX specific - const dynlibFile = "libxyz(.2|.1|).dylib" - else: - static: doAssert false -else: - static: doAssert false - -# Use cPlugin() to make any symbol changes -cPlugin: - import strutils - - # Symbol renaming examples - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - # Get rid of leading and trailing underscores - sym.name = sym.name.strip(chars = {'_'}) - - # Remove prefixes or suffixes from procs - if sym.kind == nskProc and sym.name.contains("SDL_"): - sym.name = sym.name.replace("SDL_", "") - -# Finally import wrapped header file. Recurse if #include files should also -# be wrapped. Set dynlib if binding to dynamic library. -cImport(srcDir/"include/file.h", recurse = true, dynlib="dynlibFile") diff --git a/nimterop/templite.nim b/nimterop/templite.nim deleted file mode 100644 index 7d8eb5b..0000000 --- a/nimterop/templite.nim +++ /dev/null @@ -1,36 +0,0 @@ -import os, strutils - -import nimterop/[cimport, build, paths] - -const - baseDir = currentSourcePath.parentDir()/"build" - - srcDir = baseDir/"project" - -static: - cDebug() - cDisableCaching() - - gitPull("https://github.com/user/project", outdir = srcDir, plist = """ -include/*.h -src/*.c -""", checkout = "tag/branch/hash") - - downloadUrl("https://hostname.com/file.h", outdir = srcDir) - -cIncludeDir(srcDir/"include") - -cDefine("SYMBOL", "value") - -{.passC: "flags".} -{.passL: "flags".} - -cCompile(srcDir/"file.c") - -cPlugin: - import strutils - - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - sym.name = sym.name.strip(chars = {'_'}) - -cImport(srcDir/"include/file.h", recurse = true) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 20dc79e..8695216 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -34,6 +34,7 @@ proc process(gState: State, path: string) = # CLI processing with default values proc main( check = false, + compile: seq[string] = @[], convention = "cdecl", debug = false, defines: seq[string] = @[], @@ -46,6 +47,8 @@ proc main( noComments = false, noHeader = false, output = "", + passC: seq[string] = @[], + passL: seq[string] = @[], past = false, pluginSourcePath: string = "", pnim = false, @@ -62,6 +65,7 @@ proc main( # Setup global state with arguments gState = State( + compile: compile, convention: convention, debug: debug, defines: defines, @@ -73,6 +77,8 @@ proc main( nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, + passC: passC, + passL: passL, past: past, pluginSourcePath: pluginSourcePath, pnim: pnim, @@ -220,6 +226,7 @@ when isMainModule: import cligen dispatch(main, help = { "check": "check generated wrapper with compiler", + "compile": "create {.compile.} entries in generated wrapper", "convention": "calling convention for wrapped procs", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", @@ -232,6 +239,8 @@ when isMainModule: "noComments": "exclude top-level comments from output", "noHeader": "skip {.header.} pragma in wrapper", "output": "file to output content - default: stdout", + "passC": "create {.passC.} entries in generated wrapper", + "passL": "create {.passL.} entries in generated wrapper", "past": "print AST output", "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 083d4b2..7f2b4ad 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1881,11 +1881,31 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = gState.pragmaSection.add dynPragma count += 1 - # Add `{.experimental: "codeReordering".} for #206 + # Only if not already done if gState.pragmaSection.len == count: - # Only if not already done + # Add `{.experimental: "codeReordering".} for #206 gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) + # Create `{.passC.}` from defines + for define in gState.defines: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, "-D" & define)) + + # Create `{.passC.}` from include directories + for inc in gState.includeDirs: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, "-I" & inc.quoteShell)) + + # Create `{.passC.}` from passC + for passC in gState.passC: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, passC)) + + # Create `{.passL.}` from passL + for passL in gState.passL: + gState.pragmaSection.add gState.newPragma(root, "passL", newStrNode(nkStrLit, passL)) + + # Create `{.compile.}` for specified files + for file in gState.compile: + gState.pragmaSection.add gState.newPragma(root, "compile", newStrNode(nkStrLit, file)) + proc initNim*(gState: State) = # Initialize for parseNim() one time gState.wrapperHeader = "{.push hint[ConvFromXtoItselfNotNeeded]: off.}\n" diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 09343bc..940b2df 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -22,13 +22,13 @@ when not libssh2Static: when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: + cPassL("-lpthread") + cImport(libssh2Path, recurse = true, flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} - {.passL: "-lpthread".} - assert libssh2_init(0) == 0 let diff --git a/tests/rsa.nim b/tests/rsa.nim index f5c343c..ee36286 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -35,13 +35,13 @@ cPlugin: cOverride: proc OPENSSL_die*(assertion: cstring; file: cstring; line: cint) {.importc.} +cPassL(cryptoLPath) + # Skip comments for https://github.com/tree-sitter/tree-sitter-c/issues/44 cImport(@[ basePath / "rsa.h", basePath / "err.h", ], recurse = true, flags = "-s -c " & FLAGS) -{.passL: cryptoLPath.} - OpensslInit() echo $OPENSSL_VERSION_TEXT diff --git a/tests/tast2.nim b/tests/tast2.nim index 17edfb0..a410de4 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -2,7 +2,7 @@ import macros, os, sets, strutils import nimterop/[cimport] -{.passC: "-DNIMTEROP".} +cPassC("-DNIMTEROP") static: # Skip casting on lower nim compilers because diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index 4d9a28b..d594516 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -24,15 +24,15 @@ cIncludeDir(incl) when defined(osx): cDefine("WITH_COREAUDIO") - {.passL: "-framework CoreAudio -framework AudioToolbox".} + cPassL("-framework CoreAudio -framework AudioToolbox") cCompile(src/"backend/coreaudio/*.cpp") elif defined(Linux): - {.passL: "-lpthread".} + cPassL("-lpthread") cDefine("WITH_OSS") cCompile(src/"backend/oss/*.cpp") elif defined(Windows): - {.passC: "-msse".} - {.passL: "-lwinmm".} + cPassC("-msse") + cPassL("-lwinmm") cDefine("WITH_WINMM") cCompile(src/"backend/winmm/*.cpp") else: diff --git a/tests/zlib.nim b/tests/zlib.nim index 53384c3..d4744eb 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -65,12 +65,12 @@ when zlibGit or zlibDL: when dirExists(baseDir / "buildcache"): cIncludeDir(baseDir / "buildcache") -when not zlibStatic: +when not isDefined(zlibStatic): cImport(zlibPath, recurse = true, dynlib = "zlibLPath", flags = FLAGS) else: + when isDefined(zlibJBB): + cPassL("-no-pie") + cImport(zlibPath, recurse = true, flags = FLAGS) -echo "zlib version = " & $zlibVersion() - -when isDefined(zlibJBB) and isDefined(zlibStatic): - {.passL: "-no-pie".} +echo "zlib version = " & $zlibVersion() \ No newline at end of file From aa813bdf3212f22642ec81d79f712911d9df4226 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Jul 2020 20:20:42 -0500 Subject: [PATCH 221/255] Forward getHeader passL --- nimterop/build/getheader.nim | 54 +++++++++++++++++------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index d5b398a..dbd965e 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -33,7 +33,7 @@ macro setDefines*(defs: static openArray[string]): untyped = for def in defs: let nv = def.strip().split("=", maxsplit = 1) - if nv.len != 0: + if nv.nBl: let n = nv[0] v = @@ -70,33 +70,33 @@ proc getDynlibExt(): string = proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) - if result.len != 0: + if result.nBl: break proc getStdLibPath(lname, mode: string): string = for lib in getGccLibPaths(mode): result = findFile(lname, lib, recurse = false, first = true, regex = true) - if result.len != 0: + if result.nBl: break proc getGitPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No git url setup for " & header - doAssert findExe("git").len != 0, "git executable missing" + doAssert url.nBl, "No git url setup for " & header + doAssert findExe("git").nBl, "git executable missing" gitPull(url, outdir, checkout = version) result = findFile(header, outdir) proc getDlPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No download url setup for " & header + doAssert url.nBl, "No download url setup for " & header var dlurl = url if "$#" in url or "$1" in url: - doAssert version.len != 0, "Need version for download url" + doAssert version.nBl, "Need version for download url" dlurl = url % version else: - doAssert version.len == 0, "Download url does not contain version" + doAssert version.Bl, "Download url does not contain version" downloadUrl(dlurl, outdir) @@ -107,13 +107,13 @@ proc getDlPath(header, url, outdir, version: string): string = dirname = "" break elif kind == pcDir: - if dirname.len == 0: + if dirname.Bl: dirname = path else: dirname = "" break - if dirname.len != 0: + if dirname.nBl: for kind, path in walkDir(outdir / dirname, relative = true): mvFile(outdir / dirname / path, outdir / path) @@ -124,9 +124,9 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = uri = uri if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan.io uri: " & uri + doAssert version.nBl, "Need version for Conan.io uri: " & uri uri = uri % version - elif version.len != 0: + elif version.nBl: uri = uri & "/" & version let @@ -145,12 +145,12 @@ proc getJBBPath(header, uri, flags, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] - hasVersion = version.len != 0 + hasVersion = version.nBl var ver = if spl.len == 2: spl[1] else: "" - if ver.len != 0: + if ver.nBl: if "$#" in ver or "$1" in ver: doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri ver = ver % version @@ -187,7 +187,7 @@ proc getJBBLDeps(outdir: string, shared: bool): seq[string] = result = pkg.getJBBLDeps(outdir, shared) proc getLocalPath(header, outdir: string): string = - if outdir.len != 0: + if outdir.nBl: result = findFile(header, outdir) proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = @@ -195,7 +195,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT lpath = findFile(lname, outdir, regex = true) makePath = outdir - if lpath.len != 0: + if lpath.nBl: return lpath var buildStatus: BuildStatus @@ -275,7 +275,7 @@ macro getHeader*( ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## be used for the `dynlib` parameter (within quotes) or with `cPassL()`. Any dependency ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. ## ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their @@ -285,7 +285,7 @@ macro getHeader*( ## reflect this new location. `libdir` is ignored for `Std` mode. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## will automatically add a `cPassL()` call to the static library for convenience. Note ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the ## `xxxLPath` will include paths to all of them separated by space in the right order for ## linking. @@ -335,8 +335,8 @@ macro getHeader*( name = origname.split(seps = AllChars-Letters-Digits).join() # Default to origname if not specified - conanuri = if conanuri.len != 0: conanuri else: origname - jbburi = if jbburi.len != 0: jbburi else: origname + conanuri = if conanuri.nBl: conanuri else: origname + jbburi = if jbburi.nBl: jbburi else: origname # -d:xxx for this header stdStr = name & "Std" @@ -382,10 +382,10 @@ macro getHeader*( "" mode = getCompilerMode(header) - libdir = if libdir.len != 0: libdir else: getOutDir() + libdir = if libdir.nBl: libdir else: getOutDir() # Use alternate library names if specified for regex search - if altNames.len != 0: + if altNames.nBl: lre = lre % ("(" & altNames.replace(",", "|") & ")") else: lre = lre % origname @@ -491,14 +491,12 @@ macro getHeader*( `ldeps`* = ldeps # Automatically link with static library and dependencies - {.passL: `lpath`.} - if `ldeps`.len != 0: - {.passL: `ldeps`.join(" ").} - static: gecho "# Including library " & lpath - if `ldeps`.len != 0: - gecho "# Including dependencies " & `ldeps`.join(" ") + gStateCT.passL.add lpath + if ldeps.len != 0: + gecho "# Including dependencies " & ldeps.join(" ") + gStateCT.passL.add ldeps.join(" ") else: const `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath From 44f335c87ffdf153e8e1cad4d7f59f638f19c76b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 09:08:10 -0500 Subject: [PATCH 222/255] Doc updates --- CHANGES.md | 14 ++++++++++++-- README.md | 6 +++++- nimterop/cimport.nim | 12 ++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 91cecac..8609474 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 ### Breaking changes @@ -27,6 +27,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - Nameless enum values are no longer typed to the made-up enum type name, they are instead typed as `cint` to match the underlying type. This allows using such enums without having to depend on the made-up name which could change if enum ordering changes upstream. [#236][i236] (since v0.6.1) +- Static libraries installed and linked with `getHeader()` now have their `{.passL.}` pragmas forwarded to the generated wrapper. This might lead to link errors in existing wrappers if other dependencies are specified with `{.passL.}` calls and the order of linking is wrong. This can be fixed by changing such explicit `{.passL.}` calls with `cPassL()` which will forward the link call to the generated wrapper as well. (since v0.6.5) + ### New functionality - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. @@ -43,6 +45,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. (since v0.6.4) +- All `cDefine()`, `cIncludeDir()` and `cCompile()` calls now forward relevant pragmas into the generated wrapper further enabling standalone wrappers. [#239][i239] + +- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. (since v0.6.5) + +- Added `--compile`, `--passC` and `--passL` flags to `toast` to enable the previous two improvements. (since v0.6.5) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -51,6 +59,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. (since v0.6.4) +- `cDefine()` can now accept a `seq[string]` of values. (since v0.6.5) ## Version 0.5.0 @@ -140,4 +149,5 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 [i197]: https://github.com/nimterop/nimterop/issues/197 [i200]: https://github.com/nimterop/nimterop/issues/200 [i236]: https://github.com/nimterop/nimterop/issues/236 -[i237]: https://github.com/nimterop/nimterop/issues/237 \ No newline at end of file +[i237]: https://github.com/nimterop/nimterop/issues/237 +[i239]: https://github.com/nimterop/nimterop/issues/239 \ No newline at end of file diff --git a/README.md b/README.md index cda3b5e..4770563 100644 --- a/README.md +++ b/README.md @@ -218,10 +218,12 @@ Options: -h, --help print this cligen-erated help --help-syntax advanced: prepend,plurals,.. -k, --check bool false check generated wrapper with compiler + --compile= strings {} create {.compile.} entries in generated wrapper -C=, --convention= string "cdecl" calling convention for wrapped procs -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path + -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or + file path -X=, --exclude= strings {} files or directories to exclude from the wrapped output -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor @@ -230,6 +232,8 @@ Options: -c, --noComments bool false exclude top-level comments from output -H, --noHeader bool false skip {.header.} pragma in wrapper -o=, --output= string "" file to output content - default: stdout + --passC= strings {} create {.passC.} entries in generated wrapper + --passL= strings {} create {.passL.} entries in generated wrapper -a, --past bool false print AST output --pluginSourcePath= string "" nim file to build and load as a plugin -n, --pnim bool false print Nim output diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 6120cb5..c82f884 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -368,6 +368,8 @@ macro cDefine*(name: static[string], val: static[string] = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. var str = name if val.nBl: str &= &"={val.quoteShell}" @@ -379,6 +381,8 @@ macro cDefine*(values: static seq[string]): untyped = ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. for value in values: let spl = value.split("=", maxsplit = 1) @@ -390,11 +394,15 @@ macro cDefine*(values: static seq[string]): untyped = macro cPassC*(value: static string): untyped = ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler ## during Nim compilation. + ## + ## This needs to be called before `cImport()` to take effect. gStateCT.passC.add value macro cPassL*(value: static string): untyped = ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler ## during Nim compilation. + ## + ## This needs to be called before `cImport()` to take effect. gStateCT.passL.add value proc cAddSearchDir*(dir: string) {.compileTime.} = @@ -416,6 +424,8 @@ macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): unt ## ## Set `exclude = true` if the contents of these include directories should ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. for dir in dirs: let fullpath = findPath(dir) if fullpath notin gStateCT.includeDirs: @@ -430,6 +440,8 @@ macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped ## ## Set `exclude = true` if the contents of this include directory should ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. return quote do: cIncludeDir(@[`dir`], `exclude` == 1) From e61dc54a898a395902205b5794e09ece0e9b35ba Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 11:33:26 -0500 Subject: [PATCH 223/255] renderPragma, cleanup --- CHANGES.md | 5 +- README.md | 4 +- nimterop/cimport.nim | 548 +++++++++++++++++++++++-------------------- 3 files changed, 294 insertions(+), 263 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8609474..e7e9213 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,10 +47,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - All `cDefine()`, `cIncludeDir()` and `cCompile()` calls now forward relevant pragmas into the generated wrapper further enabling standalone wrappers. [#239][i239] -- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. (since v0.6.5) +- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. These should be used in place of `{.passC.}` and `{.passL.}` and need to be called before `cImport()` to take effect. (since v0.6.5) - Added `--compile`, `--passC` and `--passL` flags to `toast` to enable the previous two improvements. (since v0.6.5) +- Added `renderPragma()` to create pragmas inline in case `cImport()` is not being used. (since v0.6.5) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -61,6 +63,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - `cDefine()` can now accept a `seq[string]` of values. (since v0.6.5) + ## Version 0.5.0 This release introduces a new backend for wrapper generation dubbed `ast2` that leverages the Nim compiler AST and renderer. The new design simplifies feature development and already includes all the functionality of the legacy algorithm plus fixes for several open issues. diff --git a/README.md b/README.md index 4770563..d652d0d 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux @@ -171,7 +171,7 @@ Nim provides some flexibility when it comes to using C/C++ libraries. In order t For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol and to `#include "header.h"` in the generated code. However, types can be mostly recreated in pure Nim so it is also possible to omit both `{.importc.}` and `{.header}` and it will work just fine except with a different name in the generated C code. This allows the user to compile the wrapper without requiring `header.h` to be present. -For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. +For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `cPassL()` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. Further, anonymous nested structs/unions within unions will be rendered incorrectly by Nim since it is unaware of the true memory structure of the type. The user will need to choose based on the library in question. diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index c82f884..88aed1a 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -64,12 +64,15 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.passC: cmd.add &" --passC+={i.quoteShell}" + gStateCT.passC = @[] for i in gStateCT.passL: cmd.add &" --passL+={i.quoteShell}" + gStateCT.passL = @[] for i in gStateCT.compile: cmd.add &" --compile+={i.sanitizePath}" + gStateCT.compile = @[] if not noNimout: cmd.add &" --pnim" @@ -142,6 +145,288 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" writeFile(cacheFile, "") +proc cDebug*() {.compileTime.} = + ## Enable debug messages and display the generated Nim code + gStateCT.debug = true + +proc cDisableCaching*() {.compileTime.} = + ## Disable caching of generated Nim code - useful during wrapper development + ## + ## If files included by header being processed by + ## `cImport()` change and affect the generated content, they will be ignored + ## and the cached value will continue to be used . Use `cDisableCaching()` to + ## avoid this scenario during development. + ## + ## `nim -f` can also be used to flush the cached content. + gStateCT.nocache = true + +proc cSearchPath*(path: string): string {.compileTime.} = + ## Get full path to file or directory `path` in search path configured + ## using `cAddSearchDir()` and `cAddStdDir()`. + ## + ## This can be used to locate files or directories that can be passed onto + ## `cCompile()`, `cIncludeDir()` and `cImport()`. + result = findPath(path, fail = false) + if result.Bl: + var found = false + for inc in gStateCT.searchDirs: + result = findPath(inc / path, fail = false) + if result.nBl: + found = true + break + doAssert found, "File or directory not found: " & path & + " gStateCT.searchDirs: " & $gStateCT.searchDirs + +proc cAddSearchDir*(dir: string) {.compileTime.} = + ## Add directory `dir` to the search path used in calls to + ## `cSearchPath()`. + runnableExamples: + import nimterop/paths, os + static: + cAddSearchDir testsIncludeDir() + doAssert cSearchPath("test.h").fileExists + + if dir notin gStateCT.searchDirs: + gStateCT.searchDirs.add(dir) + +proc cAddStdDir*(mode = "c") {.compileTime.} = + ## Add the standard `c` [default] or `cpp` include paths to search + ## path used in calls to `cSearchPath()`. + runnableExamples: + import os + static: + cAddStdDir() + doAssert cSearchPath("math.h").fileExists + for inc in getGccPaths(mode): + cAddSearchDir inc + +macro cDefine*(name: static[string], val: static[string] = ""): untyped = + ## `#define` an identifer that is forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. + var str = name + if val.nBl: + str &= &"={val.quoteShell}" + + if str notin gStateCT.defines: + gStateCT.defines.add(str) + +macro cDefine*(values: static seq[string]): untyped = + ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor + ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. + for value in values: + let + spl = value.split("=", maxsplit = 1) + name = spl[0] + val = if spl.len == 2: spl[1] else: "" + discard quote do: + cDefine(`name`, `val`) + +macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = + ## Add include directories that are forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of these include directories should + ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. + for dir in dirs: + let fullpath = findPath(dir) + if fullpath notin gStateCT.includeDirs: + gStateCT.includeDirs.add(fullpath) + if exclude: + gStateCT.exclude.add(fullpath) + +macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = + ## Add an include directory that is forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of this include directory should + ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. + return quote do: + cIncludeDir(@[`dir`], `exclude` == 1) + +macro cExclude*(paths: static seq[string]): untyped = + ## Exclude specified paths - files or directories from the wrapped output + ## + ## Full path to file or directory is required. + result = newNimNode(nnkStmtList) + for path in paths: + gStateCT.exclude.add path + +macro cExclude*(path: static string): untyped = + ## Exclude specified path - file or directory from the wrapped output. + ## + ## Full path to file or directory is required. + return quote do: + cExclude(@[`path`]) + +macro cPassC*(value: static string): untyped = + ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + ## + ## `cPassC()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + gStateCT.passC.add value + +macro cPassL*(value: static string): untyped = + ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + ## + ## `cPassL()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + gStateCT.passL.add value + +macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = + ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` + ## + ## `path` can be a specific file or contain `*` wildcard for filename: + ## + ## .. code-block:: nim + ## + ## cCompile("file.c") + ## cCompile("path/to/*.c") + ## + ## `mode` recursively searches for code files in `path`. + ## + ## `c` searches for `*.c` whereas `cpp` searches for `*.C *.cpp *.c++ *.cc *.cxx` + ## + ## .. code-block:: nim + ## + ## cCompile("path/to/dir", "cpp") + ## + ## `exclude` can be used to exclude files by partial string match. Comma separated to + ## specify multiple exclude strings + ## + ## .. code-block:: nim + ## + ## cCompile("path/to/dir", exclude="test2.c") + ## + ## `cCompile()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + + proc fcompile(file: string) = + let + (_, fn, ext) = file.splitFile() + var + ufn = fn + uniq = 1 + while ufn in gStateCT.compcache: + ufn = fn & $uniq + uniq += 1 + + # - https://github.com/nim-lang/Nim/issues/10299 + # - https://github.com/nim-lang/Nim/issues/10486 + gStateCT.compcache.add(ufn) + if fn == ufn: + gStateCT.compile.add file.replace("\\", "/") + else: + # - https://github.com/nim-lang/Nim/issues/9370 + let + hash = file.hash().abs() + tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" + if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): + cpFile(file, tmpFile) + gStateCT.compile.add tmpFile.replace("\\", "/") + + # Due to https://github.com/nim-lang/Nim/issues/9863 + # cannot use seq[string] for excludes + proc notExcluded(file, exclude: string): bool = + result = true + if "_nimterop_" in file: + result = false + elif exclude.nBl: + for excl in exclude.split(","): + if excl in file: + result = false + + proc dcompile(dir, exclude: string, ext="") = + let + (dir, pat) = + if "*" in dir: + dir.splitPath() + else: + (dir, "") + + for file in walkDirRec(dir): + if ext.nBl or pat.nBl: + let + fext = file.splitFile().ext + if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): + continue + if file.notExcluded(exclude): + fcompile(file) + + if "*" in path: + dcompile(path, exclude) + else: + let fpath = findPath(path) + if fileExists(fpath) and fpath.notExcluded(exclude): + fcompile(fpath) + elif dirExists(fpath): + if mode.contains("cpp"): + for i in @[".cpp", ".c++", ".cc", ".cxx"]: + dcompile(fpath, exclude, i) + when not defined(Windows): + dcompile(fpath, exclude, ".C") + else: + dcompile(fpath, exclude, ".c") + +macro renderPragma*(): untyped = + ## All `cDefine()`, `cIncludeDir()`, `cCompile()`, `cPassC()` and `cPassL()` + ## content typically gets forwarded via `cImport()` to the generated wrapper to be + ## rendered as part of the output so as to enable standalone wrappers. If `cImport()` + ## is not being used for some reason, `renderPragma()` can create these pragmas + ## in the nimterop wrapper itself. A good example is using `getHeader()` without + ## calling `cImport()`. + ## + ## `c2nImport()` already uses this macro so there's no need to use it when typically + ## wrapping headers. + result = newNimNode(nnkStmtList) + + for i in gStateCT.defines: + let str = "-D" & i + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.includeDirs: + let str = &"-I{i.quoteShell}" + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.passC: + result.add quote do: + {.passC: `i`.} + gStateCT.passC = @[] + + for i in gStateCT.passL: + result.add quote do: + {.passL: `i`.} + gStateCT.passL = @[] + + for i in gStateCT.compile: + result.add quote do: + {.compile: `i`.} + gStateCT.compile = @[] + +proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = + ## Similar to `cOverride()`, this macro allows filtering out symbols not of + ## interest from the generated output. + ## + ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. + runnableExamples: + static: cSkipSymbol @["proc1", "Type2"] + gStateCT.symOverride.add skips + macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not ## accurate, it may be required to hand wrap them. Define them in a `cOverride()` @@ -234,15 +519,6 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = if names.nBl: decho "Overriding " & names.join(" ") -proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = - ## Similar to `cOverride()`, this macro allows filtering out symbols not of - ## interest from the generated output. - ## - ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. - runnableExamples: - static: cSkipSymbol @["proc1", "Type2"] - gStateCT.symOverride.add skips - proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") = gStateCT.pluginSource = body @@ -332,237 +608,6 @@ macro cPluginPath*(path: static[string]): untyped = doAssert fileExists(path), "Plugin file not found: " & path cPluginHelper(readFile(path), imports = "") -proc cSearchPath*(path: string): string {.compileTime.} = - ## Get full path to file or directory `path` in search path configured - ## using `cAddSearchDir()` and `cAddStdDir()`. - ## - ## This can be used to locate files or directories that can be passed onto - ## `cCompile()`, `cIncludeDir()` and `cImport()`. - result = findPath(path, fail = false) - if result.Bl: - var found = false - for inc in gStateCT.searchDirs: - result = findPath(inc / path, fail = false) - if result.nBl: - found = true - break - doAssert found, "File or directory not found: " & path & - " gStateCT.searchDirs: " & $gStateCT.searchDirs - -proc cDebug*() {.compileTime.} = - ## Enable debug messages and display the generated Nim code - gStateCT.debug = true - -proc cDisableCaching*() {.compileTime.} = - ## Disable caching of generated Nim code - useful during wrapper development - ## - ## If files included by header being processed by - ## `cImport()` change and affect the generated content, they will be ignored - ## and the cached value will continue to be used . Use `cDisableCaching()` to - ## avoid this scenario during development. - ## - ## `nim -f` can also be used to flush the cached content. - gStateCT.nocache = true - -macro cDefine*(name: static[string], val: static[string] = ""): untyped = - ## `#define` an identifer that is forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-DXXX".}` - ## - ## This needs to be called before `cImport()` to take effect. - var str = name - if val.nBl: - str &= &"={val.quoteShell}" - - if str notin gStateCT.defines: - gStateCT.defines.add(str) - -macro cDefine*(values: static seq[string]): untyped = - ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor - ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-DXXX".}` - ## - ## This needs to be called before `cImport()` to take effect. - for value in values: - let - spl = value.split("=", maxsplit = 1) - name = spl[0] - val = if spl.len == 2: spl[1] else: "" - discard quote do: - cDefine(`name`, `val`) - -macro cPassC*(value: static string): untyped = - ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler - ## during Nim compilation. - ## - ## This needs to be called before `cImport()` to take effect. - gStateCT.passC.add value - -macro cPassL*(value: static string): untyped = - ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler - ## during Nim compilation. - ## - ## This needs to be called before `cImport()` to take effect. - gStateCT.passL.add value - -proc cAddSearchDir*(dir: string) {.compileTime.} = - ## Add directory `dir` to the search path used in calls to - ## `cSearchPath()`. - runnableExamples: - import nimterop/paths, os - static: - cAddSearchDir testsIncludeDir() - doAssert cSearchPath("test.h").fileExists - - if dir notin gStateCT.searchDirs: - gStateCT.searchDirs.add(dir) - -macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = - ## Add include directories that are forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - ## - ## Set `exclude = true` if the contents of these include directories should - ## not be included in the wrapped output. - ## - ## This needs to be called before `cImport()` to take effect. - for dir in dirs: - let fullpath = findPath(dir) - if fullpath notin gStateCT.includeDirs: - gStateCT.includeDirs.add(fullpath) - if exclude: - gStateCT.exclude.add(fullpath) - -macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = - ## Add an include directory that is forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - ## - ## Set `exclude = true` if the contents of this include directory should - ## not be included in the wrapped output. - ## - ## This needs to be called before `cImport()` to take effect. - return quote do: - cIncludeDir(@[`dir`], `exclude` == 1) - -macro cExclude*(paths: static seq[string]): untyped = - ## Exclude specified paths - files or directories from the wrapped output - ## - ## Full path to file or directory is required. - result = newNimNode(nnkStmtList) - for path in paths: - gStateCT.exclude.add path - -macro cExclude*(path: static string): untyped = - ## Exclude specified path - file or directory from the wrapped output. - ## - ## Full path to file or directory is required. - return quote do: - cExclude(@[`path`]) - -proc cAddStdDir*(mode = "c") {.compileTime.} = - ## Add the standard `c` [default] or `cpp` include paths to search - ## path used in calls to `cSearchPath()`. - runnableExamples: - import os - static: - cAddStdDir() - doAssert cSearchPath("math.h").fileExists - for inc in getGccPaths(mode): - cAddSearchDir inc - -macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = - ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` - ## - ## `path` can be a specific file or contain `*` wildcard for filename: - ## - ## .. code-block:: nim - ## - ## cCompile("file.c") - ## cCompile("path/to/*.c") - ## - ## `mode` recursively searches for code files in `path`. - ## - ## `c` searches for `*.c` whereas `cpp` searches for `*.C *.cpp *.c++ *.cc *.cxx` - ## - ## .. code-block:: nim - ## - ## cCompile("path/to/dir", "cpp") - ## - ## `exclude` can be used to exclude files by partial string match. Comma separated to - ## specify multiple exclude strings - ## - ## .. code-block:: nim - ## - ## cCompile("path/to/dir", exclude="test2.c") - - proc fcompile(file: string) = - let - (_, fn, ext) = file.splitFile() - var - ufn = fn - uniq = 1 - while ufn in gStateCT.compcache: - ufn = fn & $uniq - uniq += 1 - - # - https://github.com/nim-lang/Nim/issues/10299 - # - https://github.com/nim-lang/Nim/issues/10486 - gStateCT.compcache.add(ufn) - if fn == ufn: - gStateCT.compile.add file.replace("\\", "/") - else: - # - https://github.com/nim-lang/Nim/issues/9370 - let - hash = file.hash().abs() - tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" - if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): - cpFile(file, tmpFile) - gStateCT.compile.add tmpFile.replace("\\", "/") - - # Due to https://github.com/nim-lang/Nim/issues/9863 - # cannot use seq[string] for excludes - proc notExcluded(file, exclude: string): bool = - result = true - if "_nimterop_" in file: - result = false - elif exclude.nBl: - for excl in exclude.split(","): - if excl in file: - result = false - - proc dcompile(dir, exclude: string, ext="") = - let - (dir, pat) = - if "*" in dir: - dir.splitPath() - else: - (dir, "") - - for file in walkDirRec(dir): - if ext.nBl or pat.nBl: - let - fext = file.splitFile().ext - if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): - continue - if file.notExcluded(exclude): - fcompile(file) - - if "*" in path: - dcompile(path, exclude) - else: - let fpath = findPath(path) - if fileExists(fpath) and fpath.notExcluded(exclude): - fcompile(fpath) - elif dirExists(fpath): - if mode.contains("cpp"): - for i in @[".cpp", ".c++", ".cc", ".cxx"]: - dcompile(fpath, exclude, i) - when not defined(Windows): - dcompile(fpath, exclude, ".C") - else: - dcompile(fpath, exclude, ".c") - macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import multiple headers in one shot @@ -704,29 +749,12 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if flags.nBl: cmd.add &" {flags}" - # Have to create pragmas for c2nim since toast handles this at runtime for i in gStateCT.defines: cmd.add &" --assumedef:{i.quoteShell}" - let str = "-D" & i - result.add quote do: - {.passC: `str`.} - for i in gStateCT.includeDirs: - let str = &"-I{i.quoteShell}" - result.add quote do: - {.passC: `str`.} - - for i in gStateCT.passC: - result.add quote do: - {.passC: `i`.} - - for i in gStateCT.passL: - result.add quote do: - {.passL: `i`.} - - for i in gStateCT.compile: - result.add quote do: - {.compile: `i`.} + # Have to create pragmas for c2nim since toast handles this at runtime + result.add quote do: + renderPragma() let (c2nimout, ret) = execAction(cmd) From d95deffb6aa9993eda989a20116cf7fd72eb767a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 21:06:59 -0500 Subject: [PATCH 224/255] v0.6.5 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1eb35ef..a246483 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.4" +version = "0.6.5" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 74c9506529b422333a67e363d0554d91d542f289 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 15:33:41 -0500 Subject: [PATCH 225/255] Skip Conan and JBB deps --- CHANGES.md | 2 ++ README.md | 2 +- nimterop/build/conan.nim | 8 +++++- nimterop/build/getheader.nim | 54 ++++++++++++++++++++++++------------ nimterop/build/jbb.nim | 9 +++++- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e7e9213..edd6c97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -53,6 +53,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - Added `renderPragma()` to create pragmas inline in case `cImport()` is not being used. (since v0.6.5) +- `xxxConan` and `xxxJBB` now allow skipping required dependencies by specifying `skip=pkg1,pkg2` to the `conanFlags` and `jbbFlags` params to `getHeader()`. (since v0.6.6) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) diff --git a/README.md b/README.md index d652d0d..96f3a46 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs typically include dependencies compiled in whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. `conanFlags` and `jbbFlags` can be used to skip required dependencies from being downloaded in case another source is preferred. This can be done with `skip=pkg1,pkg2` to these flags. `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index dd43dc9..eb27557 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -22,6 +22,8 @@ type staticLibs*: seq[string] requires*: seq[ConanPackage] + skipRequires*: seq[string] + ConanBuild* = ref object ## Build type that stores build specific info and revisions bhash*: string @@ -245,7 +247,10 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = bld.options = newTable[string, string](8) for key, value in options.getFields(): bld.options[key] = value.getStr() - bld.requires = requires.to(seq[string]) + for req in requires.to(seq[string]): + # Filter skipped dependencies + if req.toLowerAscii() notin pkg.skipRequires: + bld.requires.add req bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr() if pkg.recipes.hasKey(bld.recipe_hash): @@ -422,6 +427,7 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = else: let rpkg = newConanPackageFromUri(req, shared = false) + rpkg.skipRequires = pkg.skipRequires downloadConan(rpkg, outdir, main = false) pkg.requires.add rpkg diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index dbd965e..85a0331 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -119,7 +119,7 @@ proc getDlPath(header, url, outdir, version: string): string = result = findFile(header, outdir) -proc getConanPath(header, uri, outdir, version: string, shared: bool): string = +proc getConanPath(header, uri, flags, outdir, version: string, shared: bool): string = var uri = uri @@ -131,6 +131,15 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = let pkg = newConanPackageFromUri(uri, shared) + + # Handle `conanFlags` + if flags.nBl: + for flag in flags.split(" "): + if flag.startsWith("skip="): + for req in flag["skip=".len .. ^1].split(","): + if req.nBl: + pkg.skipRequires.add req.toLowerAscii() + downloadConan(pkg, outdir) result = findFile(header, outdir) @@ -165,16 +174,21 @@ proc getJBBPath(header, uri, flags, outdir, version: string): string = # Handle `jbbFlags` if flags.nBl: - if flags.startsWith("giturl="): - let - val = flags["giturl=".len .. ^1] - if val.contains("://"): - pkg.baseUrl = val - else: - pkg.baseUrl = "https://github.com/" & val - elif flags.startsWith("url="): - pkg.baseUrl = flags["url=".len .. ^1] - pkg.isGit = false + for flag in flags.split(" "): + if flag.startsWith("giturl="): + let + val = flag["giturl=".len .. ^1] + if val.contains("://"): + pkg.baseUrl = val + else: + pkg.baseUrl = "https://github.com/" & val + elif flag.startsWith("url="): + pkg.baseUrl = flag["url=".len .. ^1] + pkg.isGit = false + elif flag.startsWith("skip="): + for req in flag["skip=".len .. ^1].split(","): + if req.nBl: + pkg.skipRequires.add req.toLowerAscii() downloadJBB(pkg, outdir) @@ -227,7 +241,7 @@ macro getHeader*( conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - jbbFlags: static[string] = "", altNames: static[string] = "", + conanFlags: static[string] = "", jbbFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or @@ -300,7 +314,12 @@ macro getHeader*( ## `cmake` and `make` in case additional configuration is required as part of the build ## process. ## - ## `jbbFlags` allows changing the BinaryBuilder.org defaults: + ## `conanFlags` and `jbbFlags` allow changing the Conan.io and BinaryBuilder.org defaults: + ## - `skip=pkg1,pkg2` skips the specified packages which are required dependencies of the + ## package in question. This enables downloading those dependencies from other sources + ## if required. + ## + ## `jbbFlags` allows two additional customizations: ## - `giturl=customUrl` changes the default `https://github.com/JuliaBinaryWrappers` to ## another Git URL. If no hostname is specified, `https://github.com` is assumed. ## - `url=customUrl` uses regular HTTP instead of Git and looks for `Artifacts.toml` and @@ -402,13 +421,14 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, jbbFlags, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, + outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) elif `nameConan`: - getConanPath(header, conanuri, outdir, version, shared) + getConanPath(header, conanuri, conanFlags, outdir, version, shared) elif `nameJBB`: getJBBPath(header, jbburi, jbbFlags, outdir, version) else: @@ -441,7 +461,7 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB @@ -477,7 +497,7 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) static: diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index a430a55..a4ff98f 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -21,6 +21,8 @@ type staticLibs*: seq[string] requires*: seq[JBBPackage] + skipRequires*: seq[string] + const # JBB URLs jbbBaseUrl = "https://github.com/JuliaBinaryWrappers" @@ -73,7 +75,12 @@ proc parseJBBProject(pkg: JBBPackage, outdir: string) = let name = line.split()[0] if name.endsWith("_jll"): - pkg.requires.add newJBBPackage(name[0 .. ^5], "") + # Filter skipped dependencies + let + pname = name[0 .. ^5] + if pname.toLowerAscii() notin pkg.skipRequires: + pkg.requires.add newJBBPackage(pname, "") + pkg.requires[^1].skipRequires = pkg.skipRequires proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = # Get build information from Artifacts.toml From 5acfd0e8197160e8f71274473a4f9f2f75bc8fbc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 16:05:22 -0500 Subject: [PATCH 226/255] Handle JBB case where no dep for os/arch combo --- nimterop/build/conan.nim | 36 ++++++++++++++++++------------------ nimterop/build/jbb.nim | 17 ++++++++++------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index eb27557..6df25d8 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -134,13 +134,13 @@ proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = proc getUriFromConanPackage*(pkg: ConanPackage): string = ## Convert a ConanPackage to a conan uri result = pkg.name - if pkg.version.len != 0: + if pkg.version.nBl: result &= "/" & pkg.version - if pkg.user.len != 0: + if pkg.user.nBl: result &= "@" & pkg.user - if pkg.channel.len != 0: + if pkg.channel.nBl: result &= "/" & pkg.channel - if pkg.bhash.len != 0: + if pkg.bhash.nBl: result &= ":" & pkg.bhash proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = @@ -149,11 +149,11 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name - if version.len != 0: + if version.nBl: query &= "/" & version - if user.len != 0: + if user.nBl: query &= "@" & user - if channel.len != 0: + if channel.nBl: query &= "/" & channel gecho &"# Searching Conan.io for latest version of {name}" @@ -172,11 +172,11 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if "@_/_" in str: let ver = str.split('/')[1].split('@')[0] - if latestv.len == 0 or compareVersions(ver, latestv) > 0: + if latestv.Bl or compareVersions(ver, latestv) > 0: latestv = ver latest = str - if latest.len != 0: + if latest.nBl: result = newConanPackageFromUri(latest) proc searchConan*(pkg: ConanPackage): ConanPackage = @@ -200,7 +200,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = vsplit[0] query = - if pkg.bhash.len == 0: + if pkg.bhash.Bl: block: var query = &"?q=arch={arch}&os={os.capitalizeAscii()}" @@ -208,7 +208,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query &= "&build_type=Release" if "shared=" notin filter: query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" - if filter.len != 0: + if filter.nBl: query &= &"&{filter}" if "compiler=" notin filter and os != "windows": query &= &"&compiler={compiler}&compiler.version=" & vfilter @@ -232,7 +232,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if not j1.isNil: for bhash, bdata in j1.getFields(): - if pkg.bhash.len == 0 or pkg.bhash == bhash: + if pkg.bhash.Bl or pkg.bhash == bhash: let bld = new(ConanBuild) settings = bdata.getOrDefault("settings") @@ -328,13 +328,13 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## Download specific `revision` of `bld` to `outdir` ## ## If omitted, the latest revision (first) is downloaded - doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() + doAssert bld.revisions.nBl, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() let outdir = fixRelPath(outdir) revision = - if revision.len != 0: + if revision.nBl: revision else: bld.revisions[0] @@ -379,7 +379,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = outdir = fixRelPath(outdir) pkg = - if pkg.version.len == 0: + if pkg.version.Bl: searchConan(pkg) else: pkg @@ -395,12 +395,12 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = pkg.getConanBuilds() - doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + doAssert pkg.recipes.nBl, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" gecho &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: - if pkg.bhash.len == 0 or pkg.bhash == build.bhash: + if pkg.bhash.Bl or pkg.bhash == build.bhash: pkg.getConanRevisions(build) pkg.dlConanBuild(build, outdir) pkg.dlConanRequires(build, outdir) @@ -442,7 +442,7 @@ proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs str = if pkg.shared: "shared" else: "static" - doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + doAssert libs.nBl, &"No {str} libs found for {pkg.name} in {outdir}" if not main: for lib in libs: diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index a4ff98f..9dacfe2 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -65,7 +65,7 @@ proc parseJBBProject(pkg: JBBPackage, outdir: string) = for line in data.splitLines(): let line = line.strip() - if line.len != 0: + if line.nBl: if line.startsWith('['): if line == "[deps]": deps = true @@ -100,7 +100,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = for line in data.splitLines(): let line = line.strip() - if line.len != 0: + if line.nBl: let spl = line.split(" = ", 1) name = spl[0] @@ -145,7 +145,7 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = quiet = true ) - if pkg.version.len != 0: + if pkg.version.nBl: # Checkout correct tag let tags = gitTags(path) @@ -157,7 +157,7 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = var url = pkg.baseUrl if "$#" in url or "$1" in url: - doAssert pkg.version.len != 0, "Need version for custom BinaryBuilder.org url: " & url + doAssert pkg.version.nBl, "Need version for custom BinaryBuilder.org url: " & url url = url % pkg.version downloadUrl(url & "Artifacts.toml", path, quiet = true) downloadUrl(url & "Project.toml", path, quiet = true) @@ -209,11 +209,14 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = pkg.getJBBRepo(outdir) - doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org" + if pkg.url.Bl: + # No url for deps means no package for that os/arch combo - e.g. Attr + doAssert not main, &"Failed to download {pkg.name} info from BinaryBuilder.org" + return let vstr = - if pkg.version.len != 0: + if pkg.version.nBl: &" v{pkg.version}" else: "" @@ -250,7 +253,7 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s libs = if shared: pkg.sharedLibs else: pkg.staticLibs str = if shared: "shared" else: "static" - doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + doAssert libs.nBl, &"No {str} libs found for {pkg.name} in {outdir}" if not main: for lib in libs: From b1ca1c6ffdbf8c2704f62c672ccfb498810970be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 17:19:41 -0500 Subject: [PATCH 227/255] Skip fix for JBB --- nimterop/build/jbb.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 9dacfe2..c8d86c3 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -260,4 +260,6 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s result.add lib for cpkg in pkg.requires: - result.add cpkg.getJBBLDeps(outdir, shared, main = false) + # No url for deps means no package for that os/arch combo - e.g. Attr + if cpkg.url.nBl: + result.add cpkg.getJBBLDeps(outdir, shared, main = false) From 5ac66286d5d44a50d0a84d1810694f818b4dda62 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 17 Jul 2020 20:56:22 -0500 Subject: [PATCH 228/255] v0.6.6 --- nimterop.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index a246483..3cb733f 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.5" +version = "0.6.6" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -48,7 +48,7 @@ task docs, "Generate docs": task minitest, "Test for Nim CI": exec "nim c -f -d:danger nimterop/toast" exec "nim c -f -d:checkAbi -r tests/tast2.nim" - exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" + exec "nim c -f -d:checkAbi -d:zlibJBB -d:zlibSetVer=1.2.11 -r tests/zlib.nim" task basic, "Basic tests": execTest "tests/tast2.nim" From dfc4e09454b5e517185b0f83c9baa642a86c6b61 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Jul 2020 14:09:48 -0500 Subject: [PATCH 229/255] Fix docs for nimble change --- nimterop/docs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 5fe87a1..6a5ef0c 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -1,4 +1,4 @@ -import macros, strformat +import strformat from os import parentDir, getCurrentCompilerExe, DirSep @@ -32,7 +32,7 @@ proc execAction(cmd: string): string = (result, ret) = gorgeEx(ccmd) doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result -proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep, +proc buildDocs*(files: openArray[string], path: string, baseDir = getCurrentDir() & $DirSep, defines: openArray[string] = @[], nimArgs = "") = ## Generate docs for all specified nim `files` to the specified `path` ## From 96d4d6a8957eed5d696e164b81353ec6b47c3a87 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 30 Jul 2020 20:33:56 -0500 Subject: [PATCH 230/255] v0.6.7 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 3cb733f..8b07a47 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.6" +version = "0.6.7" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 11814811e32ca5394bdc99c83ccaf2866b6d5c00 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 2 Aug 2020 22:20:04 -0500 Subject: [PATCH 231/255] Multiple getHeader, linkLibs name fix --- nimterop/build/getheader.nim | 8 +++++--- nimterop/build/shell.nim | 15 ++++++++++----- nimterop/toastlib/ast2.nim | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 85a0331..4549894 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -366,6 +366,7 @@ macro getHeader*( staticStr = name & "Static" verStr = name & "SetVer" + getPath = name & "GetPath" # Ident nodes of the -d:xxx to check in when statements nameStd = newIdentNode(stdStr) @@ -375,6 +376,7 @@ macro getHeader*( nameJBB = newIdentNode(jbbStr) nameStatic = newIdentNode(staticStr) + nameGetPath = newIdentNode(getPath) # Consts to generate path = newIdentNode(name & "Path") @@ -421,7 +423,7 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, + proc `nameGetPath`(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) @@ -461,7 +463,7 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, + `nameGetPath`(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB @@ -497,7 +499,7 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, + `nameGetPath`(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) static: diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f07f70c..f0af561 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -481,14 +481,19 @@ proc linkLibs*(names: openArray[string], staticLink = true): string = var stat = if staticLink: "--static" else: "" resSet: OrderedSet[string] + cmd = &"pkg-config --libs --silence-errors {stat}" resSet.init() for name in names: - let - cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" - (libs, _) = execAction(cmd, die = false) - for lib in libs.split(" "): - resSet.incl lib + for n in ["lib" & name, name]: + # Try libname and name - e.g. MagickWand doesn't have lib + let + cmd = &"{cmd} {n}" + (libs, _) = execAction(cmd, die = false) + if libs.len != 0: + for lib in libs.split(" "): + resSet.incl lib + break if staticLink: resSet.incl "--static" diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 7f2b4ad..bea6ff6 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -633,7 +633,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt yield result else: for i in start+1 ..< node.len: - if node[i].getName() == "bitfield_clause": + if node[i].getName() in ["bitfield_clause", "comment"]: continue yield gState.newIdentDef(name, node, tname, tinfo, tident, start, i, offset, exported) From 672e634d89f4db7158c9986fa96b0d5f02174259 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 2 Aug 2020 23:25:54 -0500 Subject: [PATCH 232/255] v0.6.8 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8b07a47..804f1a5 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.7" +version = "0.6.8" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From dc90a1debb8c5e914173d69f9a8d3fce2670d330 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 20:01:04 -0500 Subject: [PATCH 233/255] Fix -std issue with clang --- nimterop/treesitter/api.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 753178c..b763722 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -10,7 +10,7 @@ static: const sourcePath = cacheDir / "treesitter" / "lib" -when defined(Linux): +when defined(Linux) and defined(gcc): {.passC: "-std=c11".} {.passC: "-I$1" % (sourcePath / "include").} From f07dd4dd56c587021f6477bae3731ff4abb1f1b1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 21:12:51 -0500 Subject: [PATCH 234/255] v0.6.9 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 804f1a5..1ef3fb6 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.8" +version = "0.6.9" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From c0079db1548882949ded9fb3453a1a433a809aa8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 22:30:53 -0500 Subject: [PATCH 235/255] Test arm64 and ppc --- .travis.yml | 47 +++++++++++++++++++++++++++++++++++++---------- appveyor.yml | 4 ++-- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b58b29d..25d8449 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,47 @@ -os: - - windows - - linux - - osx +os: linux +dist: bionic +language: c addons: apt: packages: - autopoint -language: c +matrix: + include: + # Linux - amd64 + - env: BRANCH=0.20.2 + - env: BRANCH=1.0.8 + - env: BRANCH=1.2.6 + - env: BRANCH=devel -env: - - BRANCH=0.20.2 - - BRANCH=1.0.6 - - BRANCH=1.2.4 - - BRANCH=devel + # Linux - arm64 + - arch: arm64 + env: BRANCH=1.2.6 + + # Linux - ppc64 + - arch: ppc64le + env: BRANCH=1.2.6 + + # macOS - amd64 + - os: osx + env: BRANCH=0.20.2 + - os: osx + env: BRANCH=1.0.8 + - os: osx + env: BRANCH=1.2.6 + - os: osx + env: BRANCH=devel + + # windows - amd64 + - os: windows + env: BRANCH=0.20.2 + - os: windows + env: BRANCH=1.0.8 + - os: windows + env: BRANCH=1.2.6 + - os: windows + env: BRANCH=devel cache: directories: diff --git a/appveyor.yml b/appveyor.yml index b572ea1..273eee2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,8 @@ matrix: environment: matrix: - NIM_VERSION: 0.20.2 - - NIM_VERSION: 1.0.6 - - NIM_VERSION: 1.2.2 + - NIM_VERSION: 1.0.8 + - NIM_VERSION: 1.2.6 for: - From 650281039c7cb11cf089a86dc82830a347ae8784 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 16:08:11 -0500 Subject: [PATCH 236/255] Fix preprocessing bug --- nimterop/toastlib/getters.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 8a4586d..f0a4735 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -333,7 +333,8 @@ proc getPreprocessor*(gState: State, fullpath: string) = while true: if outp.readLine(line): # We want to keep blank lines here for comment processing - if line.len > 1 and line[0] == '#' and line[1] == ' ': + if line.len > 10 and line[0] == '#' and line[1] == ' ' and line.contains('"'): + # # 1 "path/to/file.h" 1 start = false line = line.split('"')[1].sanitizePath(noQuote = true) if sfile == line or From 9b848bc726ee37fb1abbce2fb51b30c189aa5a69 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 16:14:36 -0500 Subject: [PATCH 237/255] Adjust CI for failures --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25d8449..5a88169 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,16 +12,16 @@ matrix: # Linux - amd64 - env: BRANCH=0.20.2 - env: BRANCH=1.0.8 - - env: BRANCH=1.2.6 + - env: BRANCH=1.2.4 - env: BRANCH=devel # Linux - arm64 - - arch: arm64 - env: BRANCH=1.2.6 + # - arch: arm64 + # env: BRANCH=1.2.4 # Linux - ppc64 - - arch: ppc64le - env: BRANCH=1.2.6 + # - arch: ppc64le + # env: BRANCH=1.2.4 # macOS - amd64 - os: osx @@ -29,7 +29,7 @@ matrix: - os: osx env: BRANCH=1.0.8 - os: osx - env: BRANCH=1.2.6 + env: BRANCH=1.2.4 - os: osx env: BRANCH=devel @@ -39,7 +39,7 @@ matrix: - os: windows env: BRANCH=1.0.8 - os: windows - env: BRANCH=1.2.6 + env: BRANCH=1.2.4 - os: windows env: BRANCH=devel From 6e86ebc194d5f56e519802e78f9b5e39d6ea7f51 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 17:19:27 -0500 Subject: [PATCH 238/255] Fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 273eee2..d807b19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,7 +11,7 @@ environment: matrix: - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.8 - - NIM_VERSION: 1.2.6 + - NIM_VERSION: 1.2.4 for: - From f2139d53750c673208a3895e6c2b318e83ab7bf6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 17:19:31 -0500 Subject: [PATCH 239/255] v0.6.10 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1ef3fb6..1fbd440 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.9" +version = "0.6.10" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From b82d7ce87b2aa3b9ee194230565bfe6d73262b38 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Sep 2020 12:38:21 -0500 Subject: [PATCH 240/255] Fix #248 - static build on Windows --- nimterop/toast.nims | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nimterop/toast.nims b/nimterop/toast.nims index e9a3c99..e67a881 100644 --- a/nimterop/toast.nims +++ b/nimterop/toast.nims @@ -24,3 +24,7 @@ switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) # Define TOAST for globals.nim switch("define", "TOAST") + +# Static for Windows - #248 +when defined(Windows): + switch("passL", "-static") From 674d842415ef68979f6407cb31b08f179238b189 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Sep 2020 14:09:54 -0500 Subject: [PATCH 241/255] v0.6.11 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1fbd440..49c40d0 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.10" +version = "0.6.11" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 93c83728c71132a63c69ecfaab84c4c91bb05615 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 23 Aug 2020 17:11:18 -0600 Subject: [PATCH 242/255] Fix nimterop imports --- nimterop/enumtypepub.nim | 42 +++++++++++++++++++++++++++++++++++ nimterop/toastlib/getters.nim | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 nimterop/enumtypepub.nim diff --git a/nimterop/enumtypepub.nim b/nimterop/enumtypepub.nim new file mode 100644 index 0000000..7700e30 --- /dev/null +++ b/nimterop/enumtypepub.nim @@ -0,0 +1,42 @@ +import macros + +macro defineEnum*(typ: untyped): untyped = + result = newNimNode(nnkStmtList) + + # Enum mapped to distinct cint + result.add quote do: + type `typ`* = distinct cint + + for i in ["+", "-", "*", "div", "mod", "shl", "shr", "or", "and", "xor", "<", "<=", "==", ">", ">="]: + let + ni = newIdentNode(i) + typout = if i[0] in "<=>": newIdentNode("bool") else: typ # comparisons return bool + if i[0] == '>': # cannot borrow `>` and `>=` from templates + let + nopp = if i.len == 2: newIdentNode("<=") else: newIdentNode("<") + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` = `nopp`(y, x) + proc `ni`*(x: cint, y: `typ`): `typout` = `nopp`(y, x) + proc `ni`*(x, y: `typ`): `typout` = `nopp`(y, x) + else: + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` {.borrow.} + proc `ni`*(x: cint, y: `typ`): `typout` {.borrow.} + proc `ni`*(x, y: `typ`): `typout` {.borrow.} + result.add quote do: + proc `ni`*(x: `typ`, y: int): `typout` = `ni`(x, y.cint) + proc `ni`*(x: int, y: `typ`): `typout` = `ni`(x.cint, y) + + let + divop = newIdentNode("/") # `/`() + dlrop = newIdentNode("$") # `$`() + notop = newIdentNode("not") # `not`() + result.add quote do: + proc `divop`*(x, y: `typ`): `typ` = `typ`((x.float / y.float).cint) + proc `divop`*(x: `typ`, y: cint): `typ` = `divop`(x, `typ`(y)) + proc `divop`*(x: cint, y: `typ`): `typ` = `divop`(`typ`(x), y) + proc `divop`*(x: `typ`, y: int): `typ` = `divop`(x, y.cint) + proc `divop`*(x: int, y: `typ`): `typ` = `divop`(x.cint, y) + + proc `dlrop`*(x: `typ`): string {.borrow.} + proc `notop`*(x: `typ`): `typ` {.borrow.} diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index f0a4735..2e1ac4c 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -31,7 +31,7 @@ yield""".split(Whitespace).toHashSet() const # Enum macro read from file - written into wrapper when required - gEnumMacroConst = staticRead(currentSourcePath.parentDir().parentDir() / "enumtype.nim") + gEnumMacroConst = "import nimterop / enumtypepub" var gEnumMacro* = gEnumMacroConst From 00941e892588503d18b3d271a65b0596bc6d857b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 10:59:14 -0600 Subject: [PATCH 243/255] Fix regen issues windows/linux --- nimterop/build/shell.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f0af561..e5a70a8 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -206,9 +206,9 @@ proc getFileDate*(fullpath: string): string = when defined(Windows): let (head, tail) = fullpath.splitPath() - &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" + &"forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime\"" elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" + &"stat -c %Y {fullpath.sanitizePath}" elif defined(OSX) or defined(FreeBSD): &"stat -f %m {fullpath.sanitizePath}" From 316a56107a4ccfcc0773d5a5b7f59abf0e05f922 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 12:48:45 -0600 Subject: [PATCH 244/255] Add ability to get a define value --- nimterop/build/getheader.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 4549894..bbc810a 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -59,6 +59,11 @@ macro isDefined*(def: untyped): untyped = false ) +macro getDefine*(def: untyped): untyped = + if gDefines.hasKey(def.strVal()): + return newStrLitNode(gDefines[def.strVal()]) + return newStrLitNode("") + proc getDynlibExt(): string = when defined(Windows): result = "[0-9.\\-]*\\.dll" From 1b8aa46b8a5b8254cee5aa0c5d73e29881ca7ed9 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 15:24:29 -0600 Subject: [PATCH 245/255] Improve on getdefine --- nimterop/build/getheader.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index bbc810a..36dd50c 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -60,9 +60,15 @@ macro isDefined*(def: untyped): untyped = ) macro getDefine*(def: untyped): untyped = - if gDefines.hasKey(def.strVal()): - return newStrLitNode(gDefines[def.strVal()]) - return newStrLitNode("") + let version = newIdentNode(def.strVal()) + let verVal = + if gDefines.hasKey(def.strVal()): + gDefines[def.strVal()] + else: + "" + result = quote do: + const `version` {.strdefine.} = `verVal` + `version` proc getDynlibExt(): string = when defined(Windows): From 217bce386a2ef7119e4914b19461d18e0fae821a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 17 Sep 2020 19:59:41 -0600 Subject: [PATCH 246/255] Don't die on nimcheck for cached files --- nimterop/build/shell.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index e5a70a8..a8511c2 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -212,7 +212,7 @@ proc getFileDate*(fullpath: string): string = elif defined(OSX) or defined(FreeBSD): &"stat -f %m {fullpath.sanitizePath}" - (result, ret) = execAction(cmd) + (result, ret) = execAction(cmd, die=false) proc touchFile*(fullpath: string) = ## Touch file to update modified date From 60cc0d26efb79e4becb064e2db3bf5973bd25b89 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Oct 2020 17:46:27 -0500 Subject: [PATCH 247/255] Add musl support --- nimterop/build/ccompiler.nim | 4 +++- nimterop/build/conan.nim | 30 ++++++++++++++++++++++-------- nimterop/build/jbb.nim | 26 +++++++++++++++++++------- nimterop/build/shell.nim | 29 ++++++----------------------- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim index 550e1b2..f21f45a 100644 --- a/nimterop/build/ccompiler.nim +++ b/nimterop/build/ccompiler.nim @@ -75,7 +75,7 @@ proc getGccLibPaths*(mode: string): seq[string] = when defined(osx): result.add "/usr/lib" -proc getGccInfo*(): tuple[arch, os, compiler, version: string] = +proc getGccInfo*(): tuple[arch, os, compiler, version, libc: string] = let (outp, _) = execAction(&"{getCompiler()} -v") for line in outp.splitLines(): @@ -101,3 +101,5 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] = result.compiler = "clang" else: result.compiler = "gcc" + if "musl" in outp: + result.libc = "musl" diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 6df25d8..ac74089 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -16,6 +16,8 @@ type channel*: string recipes*: OrderedTableRef[string, seq[ConanBuild]] + arch*, os*, compiler*, compversion*: string + bhash*: string shared*: bool sharedLibs*: seq[string] @@ -89,6 +91,11 @@ proc `==`*(pkg1, pkg2: ConanPackage): bool = pkg1.user == pkg2.user and pkg1.channel == pkg2.channel and + pkg1.arch == pkg2.arch and + pkg1.os == pkg2.os and + pkg1.compiler == pkg2.compiler and + pkg1.compversion == pkg2.compversion and + pkg1.bhash == pkg2.bhash and pkg1.shared == pkg2.shared) @@ -101,6 +108,15 @@ proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shar result.channel = channel result.recipes = newOrderedTable[string, seq[ConanBuild]](2) + let + (arch, os, compiler, compversion, libc) = getGccInfo() + doAssert libc != "musl", "Conan does not provide precompiled binaries using musl" + + result.arch = arch + result.os = os + result.compiler = compiler + result.compversion = compversion + result.bhash = bhash result.shared = shared @@ -189,9 +205,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = ## `filter` can be used to tweak search terms ## e.g. build_type=Debug&compiler=clang let - (arch, os, compiler, version) = getGccInfo() - - vsplit = version.split('.') + vsplit = pkg.compversion.split('.') vfilter = when defined(OSX): @@ -203,18 +217,18 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if pkg.bhash.Bl: block: var - query = &"?q=arch={arch}&os={os.capitalizeAscii()}" + query = &"?q=arch={pkg.arch}&os={pkg.os.capitalizeAscii()}" if "build_type" notin filter: query &= "&build_type=Release" if "shared=" notin filter: query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" if filter.nBl: query &= &"&{filter}" - if "compiler=" notin filter and os != "windows": - query &= &"&compiler={compiler}&compiler.version=" & vfilter - if "compiler.runtime=" notin filter and os == "windows": + if "compiler=" notin filter and pkg.os != "windows": + query &= &"&compiler={pkg.compiler}&compiler.version=" & vfilter + if "compiler.runtime=" notin filter and pkg.os == "windows": query &= &"&compiler.runtime=MD" - if "compiler.version=" notin filter and os == "windows": + if "compiler.version=" notin filter and pkg.os == "windows": query &= &"&compiler.version=14" query.replace("&", "%20and%20") diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index c8d86c3..392f848 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -17,6 +17,8 @@ type url*: string # Download URL + arch*, os*, libc*: string # Target + sharedLibs*: seq[string] staticLibs*: seq[string] requires*: seq[JBBPackage] @@ -39,7 +41,11 @@ proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and pkg1.name == pkg2.name and - pkg1.version == pkg2.version) + pkg1.version == pkg2.version and + + pkg1.arch == pkg2.arch and + pkg1.os == pkg2.os and + pkg1.libc == pkg2.libc) proc newJBBPackage*(name, version: string): JBBPackage = ## Create a new JBBPackage with specified name and version @@ -49,6 +55,12 @@ proc newJBBPackage*(name, version: string): JBBPackage = result.baseUrl = jbbBaseUrl result.isGit = true + let + (arch, os, _, _, libc) = getGccInfo() + result.arch = arch + result.os = os + result.libc = libc + proc parseJBBProject(pkg: JBBPackage, outdir: string) = # Get all dependencies from Project.toml let @@ -87,8 +99,6 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = let file = outdir / jbbArtifacts - (arch, os, _, _) = getGccInfo() - if fileExists(file): let data = readFile(file) @@ -109,12 +119,14 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = # Match arch, os and glibc on Linux to find download URL case name of "arch": - if val == arch and not found: found = true + if val == pkg.arch and not found: found = true of "os": - if val != os and found: found = false + if val != pkg.os and found: found = false of "libc": when defined(Linux): - if val != "glibc" and found: found = false + if found: + let libc = if pkg.libc.nBl: pkg.libc else: "glibc" + if val != libc: found = false of "url": if found: pkg.url = val @@ -125,7 +137,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = proc findJBBLibs(pkg: JBBPackage, outdir: string) = pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) - for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): + for lib in findFiles("lib[\\\\/].*\\.(a|lib)", outdir): if not lib.endsWith(".dll.a"): pkg.staticLibs.add lib diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f0af561..d0a10fb 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -414,22 +414,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## ## Turn off recursive search with `recurse` var - cmd = - when defined(Windows): - "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" - elif defined(linux): - "find $3 $1 -regextype egrep -regex $2" - elif defined(osx) or defined(FreeBSD): - "find -E $3 $1 -regex $2" - - recursive = "" - - if recurse: - when defined(Windows): - recursive = "--recursive" - else: - when not defined(Windows): - recursive = "-maxdepth 1" + cmd = "nimgrep --follow --filenames --oneline --nocolor $1 \"$2\" $3" + recursive = if recurse: "--recursive" else: "" var dir = dir @@ -443,20 +429,17 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s file = file.extractFilename - cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] + cmd = cmd % [recursive, (".*[\\\\/]" & file & "$"), dir.sanitizePath] let (files, ret) = execAction(cmd, die = false) if ret == 0: for line in files.splitLines(): let f = - when defined(Windows): - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" + if ": " in line: + line.split(": ", maxsplit = 1)[1] else: - line + "" if f.len != 0: result.add f From 13e5ce7e221f9e675d6747857cd3f9dbef961d40 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 12:17:36 -0500 Subject: [PATCH 248/255] Add loaf, replace find --- nimterop.nimble | 6 +++++- nimterop/build/shell.nim | 16 +++++++--------- nimterop/loaf.nim | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 nimterop/loaf.nim diff --git a/nimterop.nimble b/nimterop.nimble index 49c40d0..8e12ee8 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -5,7 +5,7 @@ author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" -bin = @["nimterop/toast"] +bin = @["nimterop/toast", "nimterop/loaf"] installDirs = @["nimterop"] # Dependencies @@ -33,6 +33,9 @@ proc execTest(test: string, flags = "", runDocs = true) = task buildTimeit, "build timer": exec "nim c --hints:off -d:danger tests/timeit" +task buildLoaf, "build loaf": + execCmd("nim c --hints:off -d:danger nimterop/loaf.nim") + task buildToast, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") @@ -89,6 +92,7 @@ task test, "Test": rmFile("tests/timeit.txt") buildTimeitTask() + buildLoafTask() buildToastTask() basicTask() diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index d0a10fb..7c65b71 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -407,6 +407,9 @@ proc gitTags*(outdir: string): seq[string] = if tag.len != 0: result.add tag +proc loafExePath(): string = + currentSourcePath.parentDir.parentDir / ("loaf".addFileExt ExeExt) + proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = ## Find all matching files in the specified directory ## @@ -414,8 +417,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## ## Turn off recursive search with `recurse` var - cmd = "nimgrep --follow --filenames --oneline --nocolor $1 \"$2\" $3" - recursive = if recurse: "--recursive" else: "" + cmd = loafExePath().quoteShell & " find --rexp $1 \"$2\" $3" + recursive = if recurse: "--recurse" else: "" var dir = dir @@ -435,13 +438,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s (files, ret) = execAction(cmd, die = false) if ret == 0: for line in files.splitLines(): - let f = - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" - if f.len != 0: - result.add f + if line.len != 0: + result.add line proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = ## Find the file in the specified directory diff --git a/nimterop/loaf.nim b/nimterop/loaf.nim new file mode 100644 index 0000000..04b377e --- /dev/null +++ b/nimterop/loaf.nim @@ -0,0 +1,39 @@ +import system except find + +import os + +import cligen + +import strutils except find +import regex except find + +proc findRec(dir: string, pattern: string | Regex, recurse: bool) = + for kind, path in walkDir(dir): + if kind in [pcDir, pcLinkToDir]: + if recurse: findRec(path, pattern, recurse) + elif pattern in path: + echo path.absolutePath() + +proc find(recurse = false, rexp = false, args: seq[string]) = + var + pat = "" + rpat: Regex + for arg in args: + if not arg.startsWith("-"): + if dirExists(arg): + if rexp: + findRec(arg, rpat, recurse) + else: + findRec(arg, pat, recurse) + else: + pat = arg + if rexp: rpat = re(arg) + +when isMainModule: + dispatchMulti([ + find, help = { + "recurse": "recursive search", + "rexp": "patterns are regular expressions", + "args": "pattern1 dir1 dir2 pattern2 dir3 ..." + } + ]) From 94b3b04e6d72d1e8f02bc116abb6e8d05a90c54e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 13:33:54 -0500 Subject: [PATCH 249/255] Fix build error messaging --- nimterop/build/getheader.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 4549894..918e923 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -208,11 +208,12 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT var lpath = findFile(lname, outdir, regex = true) makePath = outdir + buildStatus: BuildStatus + errors: seq[string] if lpath.nBl: return lpath - var buildStatus: BuildStatus for buildType in buildTypes: case buildType @@ -223,6 +224,8 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT if buildStatus.built: break + elif buildStatus.error.nBl: + errors.add buildStatus.error if buildStatus.buildPath.len > 0: let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 @@ -231,7 +234,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT make(buildStatus.buildPath, lname, makeFlags, regex = true) buildStatus.built = true - let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir + let error = if errors.len > 0: errors.join("\n") else: "No build files found in " & outdir doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) From 0c2ca16f7ad9b1798f1c28ca0a3268d98e845a8d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 21:25:35 -0500 Subject: [PATCH 250/255] v0.6.12 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8e12ee8..6d4306c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.11" +version = "0.6.12" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 0e003b5dba306143beb586231f17e013a946e4b2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 8 Oct 2020 23:17:43 -0500 Subject: [PATCH 251/255] Build loaf for minitest --- nimterop.nimble | 1 + 1 file changed, 1 insertion(+) diff --git a/nimterop.nimble b/nimterop.nimble index 6d4306c..9a9083f 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -49,6 +49,7 @@ task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") task minitest, "Test for Nim CI": + exec "nim c -f -d:danger nimterop/loaf.nim" exec "nim c -f -d:danger nimterop/toast" exec "nim c -f -d:checkAbi -r tests/tast2.nim" exec "nim c -f -d:checkAbi -d:zlibJBB -d:zlibSetVer=1.2.11 -r tests/zlib.nim" From e987b50610c4756a1f5acaaba8fb865c6e0c8693 Mon Sep 17 00:00:00 2001 From: n5m <72841454+n5m@users.noreply.github.com> Date: Thu, 15 Oct 2020 13:48:39 +0000 Subject: [PATCH 252/255] ensure loaf exists before finding (#256) --- nimterop/build/shell.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 7c65b71..24075cf 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -416,8 +416,14 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## `file` is a regular expression if `regex` is true ## ## Turn off recursive search with `recurse` + let + loafExe = loafExePath() + + doAssert fileExists(loafExe), "loaf not compiled: " & loafExe.sanitizePath & + " make sure 'nimble build' or 'nimble install' built it" + var - cmd = loafExePath().quoteShell & " find --rexp $1 \"$2\" $3" + cmd = loafExe.quoteShell & " find --rexp $1 \"$2\" $3" recursive = if recurse: "--recurse" else: "" var From f7cee5c983650336f93fde5d4fea087863ac0e5e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 15 Oct 2020 08:48:57 -0500 Subject: [PATCH 253/255] v0.6.13 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 9a9083f..15a8ed1 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.12" +version = "0.6.13" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From b568863527d8b700922774e64a41db50f9b701cb Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 20 Oct 2020 11:08:52 -0600 Subject: [PATCH 254/255] Fix #233: make shl/shr use the direct left operand for type casting (#235) --- nimterop/toastlib/exprparser.nim | 16 +++++++++++++--- tests/include/tast2.h | 1 + tests/tast2.nim | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index a77dbf9..a67ed59 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -379,18 +379,28 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) result.add gState.getIdent(nimSym) let leftNode = gState.processTSNode(left, typeofNode) + var tyNode = typeofNode if typeofNode.isNil: typeofNode = nkCall.newTree( gState.getIdent("typeof"), leftNode ) + tyNode = typeofNode - let rightNode = gState.processTSNode(right, typeofNode) + # Special case of setting the shift left/right type + # to be the type of the direct left operand + if binarySym in [">>", "<<"]: + tyNode = nkCall.newTree( + gState.getIdent("typeof"), + leftNode + ) + + let rightNode = gState.processTSNode(right, tyNode) result.add leftNode result.add nkCall.newTree( - typeofNode, + tyNode, rightNode ) if binarySym == "/": @@ -399,7 +409,7 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) # So we need to emulate C here and cast the whole # expression to the type of the first arg result = nkCall.newTree( - typeofNode, + tyNode, result ) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 38c5f18..ede71d5 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,6 +35,7 @@ extern "C" { #define EQ4 AVAL < BVAL #define EQ5 AVAL != BVAL #define EQ6 AVAL == BVAL +#define SX_NEAR_ZERO (1.0f / (1 << 28)) // testing integer out of long int range #define INT_FAST16_MIN (-9223372036854775807L-1) diff --git a/tests/tast2.nim b/tests/tast2.nim index a410de4..e301fb5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -134,6 +134,8 @@ assert EQ4 == (AVAL < BVAL) assert EQ5 == (AVAL != BVAL) assert EQ6 == (AVAL == BVAL) +assert SX_NEAR_ZERO == 3.725290298461914e-09 + assert SIZEOF == 1 assert COERCE == 645635670332'u64 From fa9e66b4a371b78d97d074cf531fad8f55135334 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 15 May 2021 18:39:47 -0600 Subject: [PATCH 255/255] Update cligen --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 15a8ed1..87e1bc4 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -9,7 +9,7 @@ bin = @["nimterop/toast", "nimterop/loaf"] installDirs = @["nimterop"] # Dependencies -requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.0.0" +requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.5.3" import nimterop/docs import os