From d40a93123287b24d50166c5824373548700270c7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 16:22:53 -0600 Subject: [PATCH 1/8] Fix undefined identifiers resulting in incorrect types --- nimterop/ast2.nim | 14 ++++++++++++-- tests/include/tast2.h | 15 +++++++++++++++ tests/tast2.nim | 3 +++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4a9b048..574870f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -573,8 +573,12 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) pident = gState.getIdent(pname, pinfo, exported) result.add pident - result.add gState.getTypeArray(node[i], tident, name) - result.add newNode(nkEmpty) + let tyArray = gState.getTypeArray(node[i], tident, name) + if tyArray.kind != nkNone: + result.add tyArray + result.add newNode(nkEmpty) + else: + result = nil else: result = nil @@ -955,6 +959,8 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] + else: + result = newNode(nkNone) elif cnode.len == 1: # type name[] = UncheckedArray[type] result = gState.newArrayTree(cnode, result) @@ -985,6 +991,10 @@ proc addTypeArray(gState: State, node: TSNode) = name = typeDef.getIdentName() typ = gState.getTypeArray(node[i], tident, name) + if typ.kind == nkNone: + gecho (&"{gState.getNodeVal(node)} skipped").getCommented() + continue + typeDef.add typ # type X* = [ptr] array[x, [ptr] Y] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index bbb3315..f28c003 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -49,6 +49,21 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) +// const not supported yet +const int SOME_CONST = 8; + +struct some_struct_s +{ + int x; +}; + +struct parent_struct_s +{ + struct some_struct_s s[SOME_CONST]; +}; + +typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; + struct A0; struct A1 {}; typedef struct A2; diff --git a/tests/tast2.nim b/tests/tast2.nim index 4fd16cc..058ad0b 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -155,6 +155,9 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) +assert not compiles(parent_struct_s().s) +assert not defined(SOME_ARRAY) + assert A0 is object testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) From 97bdff30e551dea42308091765406e20de325416 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 16:32:44 -0600 Subject: [PATCH 2/8] Change to static const --- tests/include/tast2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index f28c003..d39e8ea 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -50,7 +50,7 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) // const not supported yet -const int SOME_CONST = 8; +static const int SOME_CONST = 8; struct some_struct_s { From 485770dfaf75c1f3acf7a2d3d4ad31ddb8a4b3b1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 12 May 2020 17:18:47 -0600 Subject: [PATCH 3/8] Disable windows check --- tests/include/tast2.h | 5 ++++- tests/tast2.nim | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index d39e8ea..57c060e 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -49,8 +49,10 @@ extern "C" { #define ALLSHL (SHL1 | SHL2 | SHL3) +// disable for windows for now +#ifndef _WIN32 // const not supported yet -static const int SOME_CONST = 8; +const int SOME_CONST = 8; struct some_struct_s { @@ -63,6 +65,7 @@ struct parent_struct_s }; typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; +#endif struct A0; struct A1 {}; diff --git a/tests/tast2.nim b/tests/tast2.nim index 058ad0b..f132609 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -155,8 +155,9 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) -assert not compiles(parent_struct_s().s) -assert not defined(SOME_ARRAY) +when not defined(windows): + assert not compiles(parent_struct_s().s) + assert not defined(SOME_ARRAY) assert A0 is object testFields(A0, "f1!cint") From 0deb2939bf5ec5e4b98149046b0e553f38979c41 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 17:29:26 -0600 Subject: [PATCH 4/8] Make undefined constants keep types --- nimterop/ast2.nim | 29 ++++++++++++----------------- nimterop/toast.nim | 2 ++ tests/include/tast2.h | 12 +++++------- tests/tast2.nim | 6 +++--- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 574870f..3b708b5 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -574,11 +574,8 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt pident = gState.getIdent(pname, pinfo, exported) result.add pident let tyArray = gState.getTypeArray(node[i], tident, name) - if tyArray.kind != nkNone: - result.add tyArray - result.add newNode(nkEmpty) - else: - result = nil + result.add tyArray + result.add newNode(nkEmpty) else: result = nil @@ -953,14 +950,16 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo for i in 0 ..< acount: if cnode.len == 2: # type name[X] => array[X, type] - let - # Size of array could be a Nim expression - size = gState.parseCExpression(gState.getNodeVal(cnode[1])) - if size.kind != nkNone: - result = gState.newArrayTree(cnode, result, size) - cnode = cnode[0] - else: - result = newNode(nkNone) + var size: PNode + let cnodeVal = gState.getNodeVal(cnode[1]) + # Size of array could be a Nim expression + size = gState.parseCExpression(cnodeVal) + if size.kind == nkNone: + # If the size could not be parsed, leave it alone + size = gState.getIdent(cnodeVal) + + result = gState.newArrayTree(cnode, result, size) + cnode = cnode[0] elif cnode.len == 1: # type name[] = UncheckedArray[type] result = gState.newArrayTree(cnode, result) @@ -991,10 +990,6 @@ proc addTypeArray(gState: State, node: TSNode) = name = typeDef.getIdentName() typ = gState.getTypeArray(node[i], tident, name) - if typ.kind == nkNone: - gecho (&"{gState.getNodeVal(node)} skipped").getCommented() - continue - typeDef.add typ # type X* = [ptr] array[x, [ptr] Y] diff --git a/nimterop/toast.nim b/nimterop/toast.nim index b1f910c..f54aa67 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,6 +4,8 @@ import "."/treesitter/[api, c, cpp] import "."/[ast, ast2, build, globals, getters, grammar, tshelp] +{.passC: "-DNIMTEROP".} + proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 57c060e..b8433f9 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -42,17 +42,16 @@ extern "C" { #define REG_STR "regular string" #define NOTSUPPORTEDSTR "not a " REG_STR -#define NULLCHAR '\0'/* comments should not break things*/ -#define OCTCHAR '\012' // nor should this comment +#define NULLCHAR '\0' +#define OCTCHAR '\012' #define HEXCHAR '\xFE' #define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) -// disable for windows for now -#ifndef _WIN32 -// const not supported yet -const int SOME_CONST = 8; +#ifdef NIMTEROP +#define SOME_CONST 8 +#endif struct some_struct_s { @@ -65,7 +64,6 @@ struct parent_struct_s }; typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; -#endif struct A0; struct A1 {}; diff --git a/tests/tast2.nim b/tests/tast2.nim index f132609..8deb99e 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -35,6 +35,7 @@ cOverride: type A1* = A0 +cDefine("SOME_CONST=100") cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags) proc getPragmas(n: NimNode): HashSet[string] = @@ -155,9 +156,8 @@ assert typeof(POINTERPOINTERPOINTEREXPR) is (ptr ptr ptr cint) assert ALLSHL == (SHL1 or SHL2 or SHL3) -when not defined(windows): - assert not compiles(parent_struct_s().s) - assert not defined(SOME_ARRAY) +assert typeof(parent_struct_s().s) is array[100, some_struct_s] +assert typeof(SOME_ARRAY) is array[100, some_struct_s] assert A0 is object testFields(A0, "f1!cint") From ec5ba997b75f2dba4d9dc8dc61bb4072f8286bed Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 18:31:07 -0600 Subject: [PATCH 5/8] Get rid of extra var --- nimterop/ast2.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 3b708b5..0d0883c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -573,8 +573,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) pident = gState.getIdent(pname, pinfo, exported) result.add pident - let tyArray = gState.getTypeArray(node[i], tident, name) - result.add tyArray + result.add gState.getTypeArray(node[i], tident, name) result.add newNode(nkEmpty) else: result = nil From 3bbdbeb87049ce847e3118b3b86876a67a432656 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 22:19:37 -0600 Subject: [PATCH 6/8] Preserve type array ast --- nimterop/ast2.nim | 5 +---- nimterop/exprparser.nim | 14 ++++++++------ nimterop/globals.nim | 3 +++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 0d0883c..389d371 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -952,10 +952,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo var size: PNode let cnodeVal = gState.getNodeVal(cnode[1]) # Size of array could be a Nim expression - size = gState.parseCExpression(cnodeVal) - if size.kind == nkNone: - # If the size could not be parsed, leave it alone - size = gState.getIdent(cnodeVal) + size = gState.parseCExpression(cnodeVal, skipIdentValidation = true) result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b0c34c1..e5e2150 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,14 +45,14 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - if identName notin gState.skippedSyms: + if gState.currentExprSkipIdentValidation or identName notin gState.skippedSyms: var ident = identName if ident != "_": # Process the identifier through cPlugin ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: result = gState.getIdent(ident) - elif ident.nBl and ident in gState.constIdentifiers: + elif gState.currentExprSkipIdentValidation or ident.nBl and ident in gState.constIdentifiers: if gState.currentTyCastName.nBl: ident = ident & "." & gState.currentTyCastName result = gState.getIdent(ident) @@ -591,7 +591,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = decho "NODE RESULT: ", result -proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = +proc parseCExpression*(gState: State, codeRoot: TSNode): PNode = ## Parse a c expression from a root ts node # This var is used for keeping track of the type of the first @@ -607,14 +607,16 @@ proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = decho "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) -proc parseCExpression*(gState: State, code: string, name = ""): PNode = +proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidation = false): PNode = ## Convert the C string to a nim PNode tree gState.currentExpr = code gState.currentTyCastName = name + gState.currentExprSkipIdentValidation = skipIdentValidation withCodeAst(gState.currentExpr, gState.mode): - result = gState.parseCExpression(root, name) + result = gState.parseCExpression(root) # Clear the state gState.currentExpr = "" - gState.currentTyCastName = "" \ No newline at end of file + gState.currentTyCastName = "" + gState.currentExprSkipIdentValidation = false \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 46203ea..039dd3c 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -114,6 +114,9 @@ type # Used for the exprparser.nim module currentExpr*, currentTyCastName*: string + # Controls whether or not the current expression + # should validate idents against currently defined idents + currentExprSkipIdentValidation*: bool # Legacy AST fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From 8962a851523b584e3acfb254e589c4f43415dcfb Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 13 May 2020 22:36:40 -0600 Subject: [PATCH 7/8] Cleanup code --- nimterop/ast2.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 389d371..dff355e 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -949,10 +949,9 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo for i in 0 ..< acount: if cnode.len == 2: # type name[X] => array[X, type] - var size: PNode - let cnodeVal = gState.getNodeVal(cnode[1]) - # Size of array could be a Nim expression - size = gState.parseCExpression(cnodeVal, skipIdentValidation = true) + let + # Size of array could be a Nim expression + size = gState.parseCExpression(gState.getNodeVal(cnode[1]), skipIdentValidation = true) result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] From 98beef2997d7ded26af4c8448083ba2c3909e419 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 14 May 2020 17:23:43 -0600 Subject: [PATCH 8/8] Rename currentExprSkipIdentValidation -> skipIdentValidation --- nimterop/exprparser.nim | 8 ++++---- nimterop/globals.nim | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e5e2150..b089fd2 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,14 +45,14 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - if gState.currentExprSkipIdentValidation or identName notin gState.skippedSyms: + if gState.skipIdentValidation or identName notin gState.skippedSyms: var ident = identName if ident != "_": # Process the identifier through cPlugin ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: result = gState.getIdent(ident) - elif gState.currentExprSkipIdentValidation or ident.nBl and ident in gState.constIdentifiers: + elif gState.skipIdentValidation or ident.nBl and ident in gState.constIdentifiers: if gState.currentTyCastName.nBl: ident = ident & "." & gState.currentTyCastName result = gState.getIdent(ident) @@ -611,7 +611,7 @@ proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidati ## Convert the C string to a nim PNode tree gState.currentExpr = code gState.currentTyCastName = name - gState.currentExprSkipIdentValidation = skipIdentValidation + gState.skipIdentValidation = skipIdentValidation withCodeAst(gState.currentExpr, gState.mode): result = gState.parseCExpression(root) @@ -619,4 +619,4 @@ proc parseCExpression*(gState: State, code: string, name = "", skipIdentValidati # Clear the state gState.currentExpr = "" gState.currentTyCastName = "" - gState.currentExprSkipIdentValidation = false \ No newline at end of file + gState.skipIdentValidation = false \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 039dd3c..05284a9 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -116,7 +116,7 @@ type currentExpr*, currentTyCastName*: string # Controls whether or not the current expression # should validate idents against currently defined idents - currentExprSkipIdentValidation*: bool + skipIdentValidation*: bool # Legacy AST fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string