From d2daf22df40bfee6c003030531acfc1597023181 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 07:37:17 -0600 Subject: [PATCH 01/44] Add math_expression and fix ast2 tests --- nimterop/exprparser.nim | 6 ++++-- tests/include/tast2.h | 12 ++++++++++++ tests/tast2.nim | 14 +++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f72a9a..05df0e0 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -101,7 +101,7 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -199,6 +199,8 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = nimSym = "and" of "^": nimSym = "xor" + of "+", "-", "*", "/": + nimSym = binarySym else: raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") @@ -257,7 +259,7 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node) - of "bitwise_expression": + of "bitwise_expression", "math_expression": result = exprParser.processBitwiseExpression(node) of "shift_expression": result = exprParser.processShiftExpression(node) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index bdf8823..04ecb15 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -8,6 +8,18 @@ extern "C" { #define D "hello" #define E 'c' +#define UEXPR (1234u << 1) +#define ULEXPR (1234ul << 2) +#define ULLEXPR (1234ull << 3) +#define LEXPR (1234l << 4) +#define LLEXPR (1234ll << 5) + +#define SHL1 (1u << 1) +#define SHL2 (1u << 2) +#define SHL3 (1u << 3) + +#define ALLSHL (SHL1 | SHL2 | SHL3) + struct A0; struct A1 {}; typedef struct A2; diff --git a/tests/tast2.nim b/tests/tast2.nim index e13c4ac..0749b11 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -105,6 +105,18 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert UEXPR == (1234.uint shl 1) +assert ULEXPR == (1234.uint32 shl 2) +assert ULLEXPR == (1234.uint64 shl 3) +assert LEXPR == (1234.int32 shl 4) +assert LLEXPR == (1234.int64 shl 5) + +assert SHL1 == (1.uint shl 1) +assert SHL2 == (1.uint shl 2) +assert SHL3 == (1.uint shl 3) + +assert ALLSHL == (SHL1 or SHL2 or SHL3) + assert A0 is object testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) @@ -271,7 +283,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[123 + 132, ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[123 + cast[typeof(123)](132), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From b51f0da6b252fecbf0cd9c51cf188bd548d29543 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 18 Apr 2020 09:54:55 -0600 Subject: [PATCH 02/44] Update to include more expressions --- nimterop/ast2.nim | 24 +---- nimterop/exprparser.nim | 202 ++++++++++++++++++++++++++++++---------- nimterop/getters.nim | 22 +++-- nimterop/globals.nim | 2 +- tests/include/tast2.h | 6 ++ tests/tast2.nim | 10 +- 6 files changed, 187 insertions(+), 79 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 04bb8eb..85b555f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,10 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser] +import "."/[globals, getters, exprparser, utils] proc getPtrType*(str: string): string = result = case str: @@ -17,21 +17,6 @@ proc getPtrType*(str: string): string = else: str -proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = - # Raise exception in parseString() instead of exiting for errors - if msg < warnMin: - raise newException(Exception, msgKindToString(msg)) - -proc parseString(gState: State, str: string): PNode = - # Parse a string into Nim AST - use custom error handler that raises - # an exception rather than exiting on failure - try: - result = parseString( - str, gState.identCache, gState.config, errorHandler = handleError - ) - except: - decho getCurrentExceptionMsg() - proc getLit*(nimState: NimState, str: string, expression = false): PNode = result = nimState.codeToNode(str) @@ -121,7 +106,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind != nkNilLit: + elif valident.kind != nkNone: if nimState.addNewIdentifer(name): # const X* = Y # @@ -179,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - let + var colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1501,6 +1486,7 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") + var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 05df0e0..505453c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters] +import "."/[globals, getters, utils] type ExprParser* = ref object @@ -21,15 +21,16 @@ proc newExprParser*(state: NimState, code: string): ExprParser = template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: let nimState {.inject.} = exprParser.state - necho "# " & join(msg, "").replace("\n", "\n# ") + necho join(msg, "").getCommented -template val*(node: TSNode): string = +template val(node: TSNode): string = exprParser.code.getNodeVal(node) -proc mode*(exprParser: ExprParser): string = +proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode template withCodeAst(exprParser: ExprParser, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() defer: parser.tsParserDelete() @@ -51,9 +52,9 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() - proc getNumNode(number, suffix: string): PNode {.inline.} = - result = newNode(nkNilLit) + ## Convert a C number to a Nim number PNode + result = newNode(nkNone) if number.contains("."): let floatSuffix = number[result.len-1] try: @@ -84,6 +85,8 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result = newNode(nkIntLit) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. if number.contains(re"0[xX]"): result.intVal = parseHexInt(number) result.flags = {nfBase16} @@ -97,7 +100,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) + result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch @@ -111,7 +114,7 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = getNumNode(number, suffix) - if result.kind != nkNilLit and prefix == "-": + if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( exprParser.state.getIdent("-"), result @@ -127,9 +130,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkInfix) let left = node[0] @@ -144,25 +147,29 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode result.add nkCast.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i])) + result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -179,14 +186,96 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = techo "LOG CHILD: ", child.val, ", nim: ", nimSym result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) -proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + if node.len > 1: + # Node has left and right children ie: (2 + 7) + var + res = newNode(nkInfix) + let + left = node[0] + right = node[1] + + let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + techo "MATH SYM: ", mathSym + + res.add exprParser.state.getIdent(mathSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) + + res.add leftNode + # res.add rightNode + res.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + rightNode + ) + + # Make sure the statement is of the same type as the left + # hand argument, since some expressions return a differing + # type than the input types (2/3 == float) + result = nkCall.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + res + ) + + elif node.len() == 1: + # Node has only one child, ie -(20 + 7) + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "MATH SYM: ", unarySym + + case unarySym + of "+": + nimSym = "+" + of "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gental cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + return + else: + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + else: + raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = if node.len() > 1: result = newNode(nkInfix) - let left = node[0] - let right = node[1] + + let + left = node[0] + right = node[1] + var nimSym = "" var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() @@ -199,21 +288,23 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = nimSym = "and" of "^": nimSym = "xor" - of "+", "-", "*", "/": - nimSym = binarySym else: raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) @@ -223,24 +314,26 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - techo "BIN SYM: ", binarySym + var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "BIN SYM: ", unarySym - case binarySym + case unarySym of "~": nimSym = "not" else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + ## Handle all of the types of expressions here. This proc gets called recursively + ## in the processX procs and will drill down to sub nodes. + result = newNode(nkNone) let nodeName = node.getName() techo "NODE: ", nodeName, ", VAL: ", node.val case nodeName @@ -253,36 +346,47 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = of "expression_statement", "ERROR", "translation_unit": # This may be wrong. What can be in an expression? if node.len > 0: - result = exprParser.processTSNode(node[0]) + result = exprParser.processTSNode(node[0], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") - of "parenthesized_expression": - result = exprParser.processParenthesizedExpr(node) - of "bitwise_expression", "math_expression": - result = exprParser.processBitwiseExpression(node) + result = exprParser.processParenthesizedExpr(node, typeofNode) + of "bitwise_expression": + result = exprParser.processBitwiseExpression(node, typeofNode) + of "math_expression": + result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": - result = exprParser.processShiftExpression(node) + result = exprParser.processShiftExpression(node, typeofNode) of "logical_expression": - result = exprParser.processLogicalExpression(node) + result = exprParser.processLogicalExpression(node, typeofNode) + # Why are these node types named true/false? + of "true", "false": + result = exprParser.state.parseString(node.val) of "identifier": var ident = node.val if ident != "_": + # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, nskConst) - result = exprParser.state.getIdent(ident) + techo ident + if ident != "": + result = exprParser.state.getIdent(ident) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODERES: ", result + techo "NODE RES: ", result proc codeToNode*(state: NimState, code: string): PNode = - let exprParser = newExprParser(state, code) + ## Convert the C string to a nim PNode tree + result = newNode(nkNone) try: + let exprParser = newExprParser(state, code) withCodeAst(exprParser): result = exprParser.processTSNode(root) except ExprParseError as e: - techo e.msg - result = newNode(nkNilLit) + echo e.msg.getCommented + result = newNode(nkNone) except Exception as e: - techo e.msg - result = newNode(nkNilLit) \ No newline at end of file + echo e.msg.getCommented + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b96ae70..f43f199 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -399,7 +399,7 @@ proc printLisp*(code: var string, root: TSNode): string = else: break - if node.tsNodeNamedChildCount() != 0: + if node.len() != 0: result &= "\n" nextnode = node.tsNodeNamedChild(0) depth += 1 @@ -459,15 +459,19 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(gState: State, node: TSNode) = - if gState.debug: - gecho ("Input => " & gState.getNodeVal(node)).getCommented() & "\n" & - gState.printLisp(node).getCommented() +proc printDebug*(nimState: NimState, node: TSNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug: + necho ("Input => " & nimState.getNodeVal(node)).getCommented() + necho nimState.gState.printLisp(node).getCommented() -proc printDebug*(gState: State, pnode: PNode) = - if gState.debug: - gecho ("Output => " & $pnode).getCommented() & "\n" & - gState.printTree(pnode) +proc printDebug*(nimState: NimState, pnode: PNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug and pnode.kind != nkNone: + necho ("Output => " & $pnode).getCommented() + necho nimState.printTree(pnode) # Compiler shortcuts diff --git a/nimterop/globals.nim b/nimterop/globals.nim index f159124..b964da9 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -113,7 +113,7 @@ when not declared(CIMPORT): export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl # Redirect output to file when required - template gecho*(args: string) {.dirty.} = + template gecho*(args: string) = if gState.outputHandle.isNil: echo args else: diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 04ecb15..7e15ac0 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -17,6 +17,12 @@ extern "C" { #define SHL1 (1u << 1) #define SHL2 (1u << 2) #define SHL3 (1u << 3) +#define COERCE 645635634896ull + -35436 +#define COERCE2 645635634896 + -35436 +#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define BOOL true +#define MATHEXPR (1 + 2/3*20 - 100) +#define ANDEXPR (100 & 11000) #define ALLSHL (SHL1 | SHL2 | SHL3) diff --git a/tests/tast2.nim b/tests/tast2.nim index 0749b11..d65e34a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -111,6 +111,14 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) +assert COERCE == 645635599460'u64 +assert COERCE2 == 645635599460'i64 + +assert BINEXPR == 5 +assert BOOL == true +assert MATHEXPR == -99 +assert ANDEXPR == 96 + assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) assert SHL3 == (1.uint shl 3) @@ -283,7 +291,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[123 + cast[typeof(123)](132), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 2fa669c75d2857bd1e861f3dd64a7e11c24ddadc Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 18 Apr 2020 13:13:33 -0600 Subject: [PATCH 03/44] Add sizeof expression parser, fix nkNone issue, reduce size of typeof expressions --- nimterop/ast2.nim | 2 +- nimterop/exprparser.nim | 83 +++++++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 85b555f..8c5da8c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -963,7 +963,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo let # Size of array could be a Nim expression size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) - if size.kind != nkNilLit: + if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] elif cnode.len == 1: diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 505453c..d827b06 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -130,9 +130,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -149,27 +149,26 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: P let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -189,7 +188,7 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: exprParser.processTSNode(child, typeofNode) ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) var @@ -204,19 +203,18 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo res.add exprParser.state.getIdent(mathSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode # res.add rightNode res.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -224,10 +222,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # hand argument, since some expressions return a differing # type than the input types (2/3 == float) result = nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, res ) @@ -248,6 +243,8 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # so we have to make a gental cast here to coerce it to one. # Might be bad because we are overwriting the type # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") result.add nkPrefix.newTree( exprParser.state.getIdent(unarySym), nkPar.newTree( @@ -268,7 +265,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len() > 1: result = newNode(nkInfix) @@ -288,24 +285,25 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: nimSym = "and" of "^": nimSym = "xor" + of "==", "!=": + nimSym = binarySym else: raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -330,7 +328,13 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCall.newTree( + exprParser.state.getIdent("sizeof"), + exprParser.processTSNode(node[0], typeofNode) + ) + +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) @@ -351,7 +355,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node, typeofNode) - of "bitwise_expression": + of "sizeof_expression": + result = exprParser.processSizeofExpression(node, typeofNode) + of "bitwise_expression", "equality_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) @@ -380,10 +386,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil proc codeToNode*(state: NimState, code: string): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) + # This is used for keeping track of the type of the first + # symbol + var tnode: PNode = nil try: let exprParser = newExprParser(state, code) withCodeAst(exprParser): - result = exprParser.processTSNode(root) + result = exprParser.processTSNode(root, tnode) except ExprParseError as e: echo e.msg.getCommented result = newNode(nkNone) From f061fb317936a39004a774a45a00b29b71efd7c7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 18 Apr 2020 18:58:30 -0600 Subject: [PATCH 04/44] Cleanup, fix bugs --- nimterop/exprparser.nim | 66 ++++++++++++++++++++++++++++++----------- nimterop/getters.nim | 4 +-- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index d827b06..d27f23b 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -29,6 +29,24 @@ template val(node: TSNode): string = proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode +proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + result = newNode(nkNone) + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = exprParser.state.getIdentifier(ident, kind, parent) + if ident != "": + result = exprParser.state.getIdent(ident) + +proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + exprParser.getIdent(node.val, kind, parent) + template withCodeAst(exprParser: ExprParser, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() @@ -56,7 +74,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[result.len-1] + let floatSuffix = number[number.len-1] try: case floatSuffix of 'l', 'L': @@ -66,7 +84,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = of 'f', 'F': result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: - result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1])) + result = newFloatNode(nkFloatLit, parseFloat(number)) return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") @@ -100,11 +118,12 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = + ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -149,6 +168,9 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v let leftNode = exprParser.processTSNode(left, typeofNode) + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it if typeofNode.isNil: typeofNode = nkCall.newTree( exprParser.state.getIdent("typeof"), @@ -168,6 +190,12 @@ proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) +proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCast.newTree( + exprParser.processTSNode(node[0], typeofNode), + exprParser.processTSNode(node[1], typeofNode) + ) + proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] @@ -203,6 +231,9 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var res.add exprParser.state.getIdent(mathSym) let leftNode = exprParser.processTSNode(left, typeofNode) + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it if typeofNode.isNil: typeofNode = nkCall.newTree( exprParser.state.getIdent("typeof"), @@ -212,7 +243,6 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - # res.add rightNode res.add nkCast.newTree( typeofNode, rightNode @@ -315,6 +345,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() techo "BIN SYM: ", unarySym + # TODO: Support more symbols here case unarySym of "~": nimSym = "not" @@ -363,25 +394,26 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": result = exprParser.processShiftExpression(node, typeofNode) + of "cast_expression": + result = exprParser.processCastExpression(node, typeofNode) of "logical_expression": result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "identifier": - var ident = node.val - if ident != "_": - # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, nskConst) - techo ident - if ident != "": - result = exprParser.state.getIdent(ident) + of "type_descriptor": + let ty = getType(node.val) + result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") + result = exprParser.state.getIdent(ty) + of "identifier": + result = exprParser.getIdent(node, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RES: ", result + techo "NODE RESULT: ", result proc codeToNode*(state: NimState, code: string): PNode = ## Convert the C string to a nim PNode tree @@ -389,13 +421,13 @@ proc codeToNode*(state: NimState, code: string): PNode = # This is used for keeping track of the type of the first # symbol var tnode: PNode = nil + let exprParser = newExprParser(state, code) try: - let exprParser = newExprParser(state, code) withCodeAst(exprParser): result = exprParser.processTSNode(root, tnode) except ExprParseError as e: - echo e.msg.getCommented + techo e.msg result = newNode(nkNone) except Exception as e: - echo e.msg.getCommented + techo "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f43f199..216d2e6 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -460,18 +460,16 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(nimState: NimState, node: TSNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug: necho ("Input => " & nimState.getNodeVal(node)).getCommented() necho nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug and pnode.kind != nkNone: necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode) + necho nimState.printTree(pnode).getCommented() # Compiler shortcuts From 1f56983cb2135168c4add845bcf6f888d2a1b966 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:18:04 -0600 Subject: [PATCH 05/44] Add string and char support --- nimterop/ast2.nim | 4 +- nimterop/exprparser.nim | 82 ++++++++++++++++++++++++++++++++++++----- tests/include/tast2.h | 6 +++ tests/tast2.nim | 6 +++ 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8c5da8c..290e8bf 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -164,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - var + let colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & $gState.codeToNode(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index d27f23b..833d83e 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros +import strformat, strutils, macros, sets import regex @@ -12,11 +12,12 @@ type ExprParser* = ref object state*: NimState code*: string + name*: string ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string): ExprParser = - ExprParser(state: state, code: code) +proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = + ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: @@ -38,6 +39,8 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) + if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: + ident = ident & "." & exprParser.name if ident != "": result = exprParser.state.getIdent(ident) @@ -70,6 +73,55 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() +proc parseChar(charStr: string): uint8 {.inline.} = + + if charStr.len == 1: + return charStr[0].uint8 + + # Handle octal, hex, unicode? + if charStr.startsWith("\\x"): + result = parseHexInt(charStr.replace("\\x", "0x")).uint8 + elif charStr.len == 4: # Octal + result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8 + + if result == 0: + case charStr + of "\\0": + result = ord('\0') + of "\\a": + result = 0x07 + of "\\b": + result = 0x08 + of "\\e": + result = 0x1B + of "\\f": + result = 0x0C + of "\\n": + result = '\n'.uint8 + of "\\r": + result = 0x0D + of "\\t": + result = 0x09 + of "\\v": + result = 0x0B + of "\\\\": + result = 0x5C + of "\\'": + result = '\''.uint8 + of "\\\"": + result = '\"'.uint8 + of "\\?": + result = 0x3F + else: + discard + + if result > uint8.high: + result = uint8.high + +proc getCharLit(charStr: string): PNode {.inline.} = + result = newNode(nkCharLit) + result.intVal = parseChar(charStr).int64 + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) @@ -142,12 +194,24 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkCharLit) - result.intVal = node.val[1].int64 + let val = node.val + result = getCharLit(val[1 ..< val.len - 1]) proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = - let nodeVal = node.val - result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) + let + nodeVal = node.val + strVal = nodeVal[1 ..< nodeVal.len - 1] + + const + str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + reg = re(str) + + # Convert the c string escape sequences/etc to Nim chars + var nimStr = newStringOfCap(nodeVal.len) + for m in strVal.findAll(reg): + nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) + + result = newStrNode(nkStrLit, nimStr) proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode @@ -415,13 +479,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): techo "NODE RESULT: ", result -proc codeToNode*(state: NimState, code: string): PNode = +proc codeToNode*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol var tnode: PNode = nil - let exprParser = newExprParser(state, code) + let exprParser = newExprParser(state, code, name) try: withCodeAst(exprParser): result = exprParser.processTSNode(root, tnode) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 7e15ac0..e26d660 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,6 +23,12 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) +#define CASTEXPR (char) 34 + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) diff --git a/tests/tast2.nim b/tests/tast2.nim index d65e34a..f95e1f5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -118,6 +118,12 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 +assert CASTEXPR == 34.chr + +assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" +assert NULLCHAR == '\0' +assert OCTCHAR == '\n' +assert HEXCHAR.int == 0xFE assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) From 86184ae0b6fadc4c784f7a87fb5fa8be5dc8a47c Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:23:41 -0600 Subject: [PATCH 06/44] Update some comments --- nimterop/exprparser.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 833d83e..1849996 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -74,7 +74,9 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = tree.tsTreeDelete() proc parseChar(charStr: string): uint8 {.inline.} = - + ## Parses a character literal out of a string. This is needed + ## because treesitter gives unescaped characters when parsing + ## strings. if charStr.len == 1: return charStr[0].uint8 @@ -119,6 +121,7 @@ proc parseChar(charStr: string): uint8 {.inline.} = result = uint8.high proc getCharLit(charStr: string): PNode {.inline.} = + ## Convert a character string into a proper Nim char lit node result = newNode(nkCharLit) result.intVal = parseChar(charStr).int64 From da2e698bd05a486723de21143155e546a27f5c90 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:48:48 -0600 Subject: [PATCH 07/44] Rename exprparser main proc --- nimterop/ast2.nim | 4 ++-- nimterop/exprparser.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 290e8bf..08ea2d6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -18,7 +18,7 @@ proc getPtrType*(str: string): string = str proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.codeToNode(str) + result = nimState.parseCExpression(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & $gState.codeToNode(gState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1849996..352fe18 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -482,7 +482,7 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): techo "NODE RESULT: ", result -proc codeToNode*(state: NimState, code: string, name = ""): PNode = +proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first From 6ff4e1d459152595262042144d4ca5dd836c5197 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:50:04 -0600 Subject: [PATCH 08/44] Don't export parse procs --- nimterop/exprparser.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 352fe18..183bb18 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -172,7 +172,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result.intVal = parseInt(number) -proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -196,11 +196,11 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = let val = node.val result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -216,9 +216,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = newStrNode(nkStrLit, nimStr) -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -252,18 +252,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" From 4ffbc33ab35c6bf1344291d52b4d62264db5fc6b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:56:32 -0600 Subject: [PATCH 09/44] Add missing utils module --- nimterop/utils.nim | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 nimterop/utils.nim diff --git a/nimterop/utils.nim b/nimterop/utils.nim new file mode 100644 index 0000000..025256a --- /dev/null +++ b/nimterop/utils.nim @@ -0,0 +1,18 @@ +import compiler/[ast, lineinfos, msgs, options, parser, renderer] + +import "."/[globals, getters] + +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting for errors + if msg < warnMin: + raise newException(Exception, msgKindToString(msg)) + +proc parseString*(nimState: NimState, str: string): PNode = + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + decho getCurrentExceptionMsg() \ No newline at end of file From ff57369fcec3fb491340304333e357298dcf8291 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 08:31:45 -0600 Subject: [PATCH 10/44] Try to fix array type test --- tests/tast2.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tast2.nim b/tests/tast2.nim index f95e1f5..d00552c 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -93,11 +93,11 @@ macro testFields(t: typed, fields: static[string] = "") = for i in 0 ..< rl.len: let name = ($rl[i][0]).strip(chars = {'*'}) - typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "") + typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "").replace("typeof", "type") n = names.find(name) assert n != -1, $t & "." & name & " invalid" - assert types[n] == typ, - "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ + assert types[n].replace("typeof", "type") == typ, + "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ assert A == 2 assert B == 1.0 From d082f57ce2215a834c896dd60ed81fb964ac6f31 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 08:45:20 -0600 Subject: [PATCH 11/44] Try fix cast test error --- tests/include/tast2.h | 2 +- tests/tast2.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e26d660..3c6148a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,7 +23,7 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -#define CASTEXPR (char) 34 +#define CASTEXPR (int) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index d00552c..be28230 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -118,7 +118,7 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -assert CASTEXPR == 34.chr +assert CASTEXPR == 34 assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' From ca4380693e7ee451ed941f301709e530246af29c Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 08:49:29 -0600 Subject: [PATCH 12/44] Disable cast test for now --- tests/include/tast2.h | 2 +- tests/tast2.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 3c6148a..6af6db0 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,7 +23,7 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -#define CASTEXPR (int) 34 +//define CASTEXPR (int) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index be28230..80713b0 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -118,7 +118,7 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -assert CASTEXPR == 34 +# assert CASTEXPR == 34.chr assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' From e62d204d574a41dd3d7525b101491450265b4d97 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 08:53:00 -0600 Subject: [PATCH 13/44] Revert back comment test. Have to figure out how to test without vm --- tests/include/tast2.h | 2 +- tests/tast2.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 6af6db0..3c6148a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,7 +23,7 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -//define CASTEXPR (int) 34 +#define CASTEXPR (int) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index 80713b0..d00552c 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -118,7 +118,7 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -# assert CASTEXPR == 34.chr +assert CASTEXPR == 34.chr assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' From 9c4386b7fd9af993d1576826c5e4a36ec0f2aa1b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 09:08:08 -0600 Subject: [PATCH 14/44] Fix cast type --- tests/include/tast2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 3c6148a..e26d660 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,7 +23,7 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -#define CASTEXPR (int) 34 +#define CASTEXPR (char) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' From 1953bff61b291c62e5202a4c484f9e10b3f68a70 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 09:19:36 -0600 Subject: [PATCH 15/44] only run tast2 on supported compilers --- nimterop.nimble | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index a6fba1e..e2f82b1 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -37,8 +37,9 @@ task docs, "Generate docs": task test, "Test": buildToastTask() - execTest "tests/tast2.nim" - execTest "tests/tast2.nim", "-d:HEADER" + when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0): + execTest "tests/tast2.nim" + execTest "tests/tast2.nim", "-d:HEADER" execTest "tests/tnimterop_c.nim" execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" From bd3d4e42d76ee737652419005051d0810f2a42a8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 11:30:29 -0600 Subject: [PATCH 16/44] Change cast to gentle type cast --- nimterop/exprparser.nim | 6 +++--- tests/include/tast2.h | 4 ++-- tests/tast2.nim | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 183bb18..ed287a3 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -247,7 +247,7 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( typeofNode, rightNode ) @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCast.newTree( + result = nkCall.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) @@ -310,7 +310,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - res.add nkCast.newTree( + res.add nkCall.newTree( typeofNode, rightNode ) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e26d660..53b4d64 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -17,8 +17,8 @@ extern "C" { #define SHL1 (1u << 1) #define SHL2 (1u << 2) #define SHL3 (1u << 3) -#define COERCE 645635634896ull + -35436 -#define COERCE2 645635634896 + -35436 +#define COERCE 645635634896ull + 35436 +#define COERCE2 645635634896 + 35436ul #define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) diff --git a/tests/tast2.nim b/tests/tast2.nim index d00552c..d43ce9f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -111,8 +111,8 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert COERCE == 645635599460'u64 -assert COERCE2 == 645635599460'i64 +assert COERCE == 645635670332'u64 +assert COERCE2 == 645635670332'i64 assert BINEXPR == 5 assert BOOL == true @@ -297,7 +297,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 97fb1d462010fa2f7a3e74b76e22af3b00f985d5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 11:31:16 -0600 Subject: [PATCH 17/44] Re enable tests for Nim < 1.0.0 --- nimterop.nimble | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index e2f82b1..a6fba1e 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -37,9 +37,8 @@ task docs, "Generate docs": task test, "Test": buildToastTask() - when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0): - execTest "tests/tast2.nim" - execTest "tests/tast2.nim", "-d:HEADER" + execTest "tests/tast2.nim" + execTest "tests/tast2.nim", "-d:HEADER" execTest "tests/tnimterop_c.nim" execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" From ac0913298e14d31063a41ce7dc412a6b1ddeb738 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 11:33:53 -0600 Subject: [PATCH 18/44] Make bitwise_expression forwards compatible --- nimterop/exprparser.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index ed287a3..dd664c6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -455,7 +455,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": result = exprParser.processSizeofExpression(node, typeofNode) - of "bitwise_expression", "equality_expression": + # binary_expression from the new treesitter upgrade should work here + # once we upgrade + of "bitwise_expression", "equality_expression", "binary_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) From 03e8a99ed3011b0295711edc849718675f05718a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 12:21:52 -0600 Subject: [PATCH 19/44] Skip syms for tmath --- tests/tmath.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tmath.nim b/tests/tmath.nim index 5d84700..83d598e 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,6 +13,7 @@ when defined(windows): complex = object static: + cSkipSymbols = @["mingw_choose_expr", "EXCEPTION_DEFINED"] cDebug() cDisableCaching() cAddStdDir() From 26971ff26deeed61333e845fa6ab8db209ffeb8e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 12:23:45 -0600 Subject: [PATCH 20/44] Skip more syms --- tests/tmath.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tmath.nim b/tests/tmath.nim index 83d598e..fdf3a76 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,7 +13,7 @@ when defined(windows): complex = object static: - cSkipSymbols = @["mingw_choose_expr", "EXCEPTION_DEFINED"] + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] cDebug() cDisableCaching() cAddStdDir() From ac646a9feaadd2ec55bc402529e9d5bafa42860b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 13:38:27 -0600 Subject: [PATCH 21/44] Add casting back for cast_expression. Disable cast test on Nim < 1.0.0 --- nimterop/exprparser.nim | 2 +- tests/tast2.nim | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index dd664c6..a944547 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCall.newTree( + result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) diff --git a/tests/tast2.nim b/tests/tast2.nim index d43ce9f..8dcdc0f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -3,11 +3,16 @@ import macros, os, sets, strutils import nimterop/[cimport] static: + # Skip casting on lower nim compilers because + # the VM does not support it + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["CASTEXPR"] cDebug() const path = currentSourcePath.parentDir() / "include" / "tast2.h" + when defined(HEADER): cDefine("HEADER") const @@ -118,7 +123,9 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -assert CASTEXPR == 34.chr + +when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0): + assert CASTEXPR == 34.chr assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' From 1b8ba144d73250b7349b133f4f8871907ab82fb8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 14:17:17 -0600 Subject: [PATCH 22/44] Skip more syms for cast expressions on Nim < 1.0.0 --- nimterop/exprparser.nim | 2 +- tests/tast2.nim | 2 +- tests/tmath.nim | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a944547..207d7fd 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -499,4 +499,4 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + result = newNode(nkNone) diff --git a/tests/tast2.nim b/tests/tast2.nim index 8dcdc0f..0b3762a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -460,4 +460,4 @@ checkPragmas(nested, pHeaderImpBy) when defined(HEADER): assert sitest1(5) == 10 - assert sitest1(10) == 20 \ No newline at end of file + assert sitest1(10) == 20 diff --git a/tests/tmath.nim b/tests/tmath.nim index fdf3a76..bbf7abd 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,7 +13,10 @@ when defined(windows): complex = object static: - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] + else: + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] cDebug() cDisableCaching() cAddStdDir() From 6158aec0ae31c3b77614e83ec8bd798b5636269e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:09:08 -0600 Subject: [PATCH 23/44] Use tsNodeChild --- nimterop/exprparser.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 207d7fd..610c0cb 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -223,7 +223,8 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let left = node[0] right = node[1] - var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + + let shiftSym = node.tsNodeChild(1).val.strip() case shiftSym of "<<": @@ -268,7 +269,7 @@ proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(0).val.strip() techo "LOG SYM: ", binarySym case binarySym @@ -292,7 +293,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var left = node[0] right = node[1] - let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let mathSym = node.tsNodeChild(1).val.strip() techo "MATH SYM: ", mathSym res.add exprParser.state.getIdent(mathSym) @@ -329,7 +330,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let child = node[0] var nimSym = "" - let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "MATH SYM: ", unarySym case unarySym @@ -372,7 +373,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var nimSym = "" - var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(1).val.strip() techo "BIN SYM: ", binarySym case binarySym @@ -409,7 +410,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym # TODO: Support more symbols here @@ -437,7 +438,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() + techo "NODE: ", nodeName, ", VAL: ", node.val + case nodeName of "number_literal": result = exprParser.processNumberLiteral(node) From de079bb9f0443ef8d2e7f0c18d9df192b8e7e0d0 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:55:05 -0600 Subject: [PATCH 24/44] Add comments describing capabilities --- nimterop/exprparser.nim | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 610c0cb..1fc72ab 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -8,6 +8,27 @@ import "."/treesitter/[api, c, cpp] import "."/[globals, getters, utils] +# This version of exprparser should be able to handle: +# +# All integers + integer like expressions (hex, octal, suffixes) +# All floating point expressions (except for C++'s hex floating point stuff) +# Strings and character literals, including C's escape characters (not sure if this is the same as C++'s escape characters or not) +# Math operators (+, -, /, *) +# Some Unary operators (-, !, ~). ++, --, and & are yet to be implemented +# Any identifiers +# C type descriptors (int, char, etc) +# Boolean values (true, false) +# Shift expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Math expressions (containing anything in this list) +# Sizeof expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Parentheses expressions (containing anything in this list) +# Expressions containing other expressions +# +# In addition to the above, it should also handle most type coercions, except +# for where Nim can't (such as uint + -int) + type ExprParser* = ref object state*: NimState @@ -413,7 +434,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym - # TODO: Support more symbols here + # TODO: Support more symbols here. ++, --, & case unarySym of "~": nimSym = "not" From a2b1acc5c80aa1760e31631d8b1da9cb45dc3a0e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 18:04:12 -0600 Subject: [PATCH 25/44] Add missing sized type specifier for 'long int', etc --- nimterop/exprparser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1fc72ab..44e59d6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -494,7 +494,7 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "type_descriptor": + of "type_descriptor", "sized_type_specifier": let ty = getType(node.val) result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: From 9bd23ab7164af1328ac5847f090a2d803e9414f3 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 19:01:53 -0600 Subject: [PATCH 26/44] Fix pcre tests --- tests/tpcre.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tpcre.nim b/tests/tpcre.nim index c8e8059..3bbd1ba 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,6 +7,7 @@ const pcreH = baseDir/"pcre.h.in" static: + cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From 0d9f1ddc87849e380edf5692d65c49ea5222d1f2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 01:16:19 -0600 Subject: [PATCH 27/44] Update based on comments from review. Need to add more docs and reorg to use gstate --- nimterop/ast2.nim | 30 +- nimterop/{utils.nim => comphelp.nim} | 0 nimterop/exprparser.nim | 394 ++++++++++++--------------- nimterop/getters.nim | 20 +- nimterop/toast.nim | 20 +- nimterop/tshelp.nim | 30 ++ tests/tast2.nim | 1 - 7 files changed, 249 insertions(+), 246 deletions(-) rename nimterop/{utils.nim => comphelp.nim} (100%) create mode 100644 nimterop/tshelp.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 08ea2d6..1b7fb3d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,12 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times +import options as opts + import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser, utils] +import "."/[globals, getters, exprparser, comphelp] proc getPtrType*(str: string): string = result = case str: @@ -17,9 +19,6 @@ proc getPtrType*(str: string): string = else: str -proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.parseCExpression(str) - proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined # and use that if present @@ -101,7 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gState.getNodeVal(node[1]) valident = - gState.getLit(val) + gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank @@ -962,7 +961,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo # type name[X] => array[X, type] let # Size of array could be a Nim expression - size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) + size = gState.parseCExpression(gState.getNodeVal(cnode[1])) if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] @@ -1367,6 +1366,7 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] + fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1385,20 +1385,25 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - # Explicit value - fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name - - # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + fvalSections.add((fname, "", some(en[1]))) + else: + fvalSections.add((fname, fval, none(TSNode))) fnames.incl fname - prev = fname # Add fields to list of consts after processing enum so that we don't cast # enum field to itself gState.constIdentifiers.incl fnames + # parseCExpression requires all const identifiers to be present for the enum + for (fname, fval, cexprNode) in fvalSections: + var fval = fval + if cexprNode.isSome: + fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) @@ -1486,7 +1491,6 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") - var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/utils.nim b/nimterop/comphelp.nim similarity index 100% rename from nimterop/utils.nim rename to nimterop/comphelp.nim diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 44e59d6..2ce4989 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, utils] +import "."/[globals, getters, comphelp, tshelp] # This version of exprparser should be able to handle: # @@ -41,9 +41,9 @@ proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = - if exprParser.state.gState.debug: + block: let nimState {.inject.} = exprParser.state - necho join(msg, "").getCommented + decho join(msg, "") template val(node: TSNode): string = exprParser.code.getNodeVal(node) @@ -60,9 +60,11 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) - if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: - ident = ident & "." & exprParser.name - if ident != "": + if kind == nskType: + result = exprParser.state.getIdent(ident) + elif ident.nBl and ident in exprParser.state.constIdentifiers: + if exprParser.name.nBl: + ident = ident & "." & exprParser.name result = exprParser.state.getIdent(ident) proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = @@ -71,29 +73,6 @@ proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = "" ## Returns PNode(nkNone) if the identifier is blank exprParser.getIdent(node.val, kind, parent) -template withCodeAst(exprParser: ExprParser, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code - var parser = tsParserNew() - defer: - parser.tsParserDelete() - - doAssert exprParser.code.nBl, "Empty code" - if exprParser.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif exprParser.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {exprParser.mode}" - - var - tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32) - root {.inject.} = tree.tsTreeRootNode() - - body - - defer: - tree.tsTreeDelete() - proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed ## because treesitter gives unescaped characters when parsing @@ -161,37 +140,36 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: result = newFloatNode(nkFloatLit, parseFloat(number)) - return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") - - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) else: - result = newNode(nkIntLit) + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc @@ -285,168 +263,112 @@ proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let binarySym = node.tsNodeChild(0).val.strip() - techo "LOG SYM: ", binarySym - - case binarySym - of "!": - nimSym = "not" +proc getNimUnarySym(csymbol: string): string = + ## Get the Nim equivalent of a unary C symbol + ## + ## TODO: Add ++, --, + case csymbol + of "+", "-": + result = csymbol + of "~", "!": + result = "not" else: - raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{csymbol}\"") - techo "LOG CHILD: ", child.val, ", nim: ", nimSym - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) +proc getNimBinarySym(csymbol: string): string = + case csymbol + of "|", "||": + result = "or" + of "&", "&&": + result = "and" + of "^": + result = "xor" + of "==", "!=", + "+", "-", "/", "*", + ">", "<", ">=", "<=": + result = csymbol + of "%": + result = "mod" + else: + raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") + +proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + # Node has left and right children ie: (2 + 7) + result = newNode(nkInfix) + + let + left = node[0] + right = node[1] + binarySym = node.tsNodeChild(1).val.strip() + nimSym = getNimBinarySym(binarySym) + + result.add exprParser.state.getIdent(nimSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) + + let rightNode = exprParser.processTSNode(right, typeofNode) + + result.add leftNode + result.add nkCall.newTree( + typeofNode, + rightNode ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = newNode(nkPar) + + let + child = node[0] + unarySym = node.tsNodeChild(0).val.strip() + nimSym = getNimUnarySym(unarySym) + + if nimSym == "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gentle cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + else: + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + +proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) - var - res = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let mathSym = node.tsNodeChild(1).val.strip() - techo "MATH SYM: ", mathSym - - res.add exprParser.state.getIdent(mathSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - res.add leftNode - res.add nkCall.newTree( - typeofNode, - rightNode - ) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) + let binExpr = processBinaryExpression(exprParser, node, typeofNode) + # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, - res + binexpr ) elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "MATH SYM: ", unarySym - - case unarySym - of "+": - nimSym = "+" - of "-": - # Special case. The minus symbol must be in front of an integer, - # so we have to make a gental cast here to coerce it to one. - # Might be bad because we are overwriting the type - # There's probably a better way of doing this - if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") - result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), - nkPar.newTree( - nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) - ) - ) - ) - return - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) + result = processUnaryExpression(exprParser, node, typeofNode) else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") - -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - if node.len() > 1: - result = newNode(nkInfix) - - let - left = node[0] - right = node[1] - - var nimSym = "" - - let binarySym = node.tsNodeChild(1).val.strip() - techo "BIN SYM: ", binarySym - - case binarySym - of "|", "||": - nimSym = "or" - of "&", "&&": - nimSym = "and" - of "^": - nimSym = "xor" - of "==", "!=": - nimSym = binarySym - else: - raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") - - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - - elif node.len() == 1: - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "BIN SYM: ", unarySym - - # TODO: Support more symbols here. ++, --, & - case unarySym - of "~": - nimSym = "not" - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) - else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( @@ -464,45 +386,85 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): case nodeName of "number_literal": + # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f + # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 result = exprParser.processNumberLiteral(node) of "string_literal": + # Input -> "foo\0\x42" + # Output -> "foo\0" result = exprParser.processStringLiteral(node) of "char_literal": + # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> result = exprParser.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": - # This may be wrong. What can be in an expression? - if node.len > 0: + # Note that we're parsing partial expressions, so the TSNode might contain + # an ERROR node. If that's the case, they usually contain children with + # partial results, which will contain parsed expressions + # + # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) + # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ + if node.len == 1: result = exprParser.processTSNode(node[0], typeofNode) + elif node.len > 1: + result = newNode(nkStmtListExpr) + for i in 0 ..< node.len: + result.add exprParser.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": + # Input -> (IDENT - OTHERIDENT) + # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": + # Input -> sizeof(char) + # Output -> sizeof(cchar) result = exprParser.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade - of "bitwise_expression", "equality_expression", "binary_expression": - result = exprParser.processBitwiseExpression(node, typeofNode) - of "math_expression": - result = exprParser.processMathExpression(node, typeofNode) + of "math_expression", "logical_expression", "relational_expression", + "bitwise_expression", "equality_expression", "binary_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + # Output -> + # typeof(a)(a == typeof(a)(b)) + # typeof(a)(a != typeof(a)(b)) + # (not a) + # (not a) + # typeof(a)(a < typeof(a)(b)) + # typeof(a)(a > typeof(a)(b)) + # typeof(a)(a <= typeof(a)(b)) + # typeof(a)(a >= typeof(a)(b)) + result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": + # Input -> a >> b, a << b + # Output -> a shr typeof(a)(b), a shl typeof(a)(b) result = exprParser.processShiftExpression(node, typeofNode) of "cast_expression": + # Input -> (int) a + # Output -> cast[cint](a) result = exprParser.processCastExpression(node, typeofNode) - of "logical_expression": - result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": + # Input -> true, false + # Output -> true, false result = exprParser.state.parseString(node.val) of "type_descriptor", "sized_type_specifier": + # Input -> int, unsigned int, long int, etc + # Output -> cint, cuint, clong, etc let ty = getType(node.val) - result = exprParser.getIdent(ty, nskType, parent=node.getName()) - if result.kind == nkNone: - result = exprParser.state.getIdent(ty) + if ty.len > 0: + # If ty is not empty, one of C's builtin types has been found + result = exprParser.getIdent(ty, nskType, parent=node.getName()) + else: + result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": + # Input -> IDENT + # Output -> IDENT (if found in sym table, else error) result = exprParser.getIdent(node, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") + raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") @@ -512,15 +474,15 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first - # symbol + # symbol used for type casting var tnode: PNode = nil let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser): + withCodeAst(exprParser.code, exprParser.mode): result = exprParser.processTSNode(root, tnode) except ExprParseError as e: techo e.msg result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 216d2e6..b7744ce 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -401,7 +401,7 @@ proc printLisp*(code: var string, root: TSNode): string = if node.len() != 0: result &= "\n" - nextnode = node.tsNodeNamedChild(0) + nextnode = node[0] depth += 1 else: result &= ")\n" @@ -432,7 +432,7 @@ proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") proc printTree*(gState: State, pnode: PNode, offset = ""): string = - if gState.debug and pnode.kind != nkNone: + if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: @@ -459,17 +459,17 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(nimState: NimState, node: TSNode) = +proc printDebug*(gState: State, node: TSNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug: - necho ("Input => " & nimState.getNodeVal(node)).getCommented() - necho nimState.gState.printLisp(node).getCommented() + if gState.debug: + gecho ("Input => " & gState.getNodeVal(node)).getCommented() + gecho gState.printLisp(node).getCommented() -proc printDebug*(nimState: NimState, pnode: PNode) = +proc printDebug*(gState: State, pnode: PNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug and pnode.kind != nkNone: - necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode).getCommented() + if gState.debug and pnode.kind != nkNone: + gecho ("Output => " & $pnode).getCommented() + gecho gState.printTree(pnode).getCommented() # Compiler shortcuts diff --git a/nimterop/toast.nim b/nimterop/toast.nim index ab44970..afb511d 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,16 +2,11 @@ import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, globals, getters, grammar, build] +import "."/[ast, ast2, globals, getters, grammar, build, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" - var parser = tsParserNew() - - defer: - parser.tsParserDelete() - if gState.mode.Bl: gState.mode = getCompilerMode(path) @@ -20,6 +15,7 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) +<<<<<<< HEAD doAssert gState.code.nBl, "Empty file or preprocessor error" if gState.mode == "c": @@ -45,6 +41,18 @@ proc process(gState: State, path: string, astTable: AstTable) = ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code +======= + withCodeAst(gState.code, gState.mode): + if gState.past: + gecho gState.printLisp(root) + elif gState.pnim: + if Feature.ast2 in gState.feature: + ast2.printNim(gState, path, root) + else: + ast.printNim(gState, path, root, astTable) + elif gState.preprocess: + gecho gState.code +>>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim new file mode 100644 index 0000000..f234bc0 --- /dev/null +++ b/nimterop/tshelp.nim @@ -0,0 +1,30 @@ +template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + + # This section is needed to be able to reference + # mode in strformat calls + let + code = inputCode + mode {.inject.} = inputMode + + var parser = tsParserNew() + defer: + parser.tsParserDelete() + + doAssert code.nBl, "Empty code or preprocessor error" + + if mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {mode}" + + var + tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) + root {.inject.} = tree.tsTreeRootNode() + + body + + defer: + tree.tsTreeDelete() \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index 0b3762a..e82430b 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -12,7 +12,6 @@ static: const path = currentSourcePath.parentDir() / "include" / "tast2.h" - when defined(HEADER): cDefine("HEADER") const From 1b4591df35a0ca4fd74e7dafd8db72e63b01f370 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 20:20:34 -0600 Subject: [PATCH 28/44] nimState -> gState --- nimterop/ast2.nim | 8 +- nimterop/comphelp.nim | 4 +- nimterop/exprparser.nim | 247 ++++++++++++++++++++-------------------- nimterop/globals.nim | 9 +- nimterop/toast.nim | 32 +----- 5 files changed, 136 insertions(+), 164 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 1b7fb3d..4fec237 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -104,9 +104,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, nskConst) + result = gState.getOverrideOrSkip(node, origname, nskConst) elif valident.kind != nkNone: - if nimState.addNewIdentifer(name): + if gState.addNewIdentifer(name): # const X* = Y # # nkConstDef( @@ -1400,9 +1400,9 @@ proc addEnum(gState: State, node: TSNode) = for (fname, fval, cexprNode) in fvalSections: var fval = fval if cexprNode.isSome: - fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] # Add other names if node.getName() == "type_definition" and node.len > 1: diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim index 025256a..1709f8b 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/comphelp.nim @@ -7,12 +7,12 @@ proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) if msg < warnMin: raise newException(Exception, msgKindToString(msg)) -proc parseString*(nimState: NimState, str: string): PNode = +proc parseString*(gState: State, str: string): PNode = # Parse a string into Nim AST - use custom error handler that raises # an exception rather than exiting on failure try: result = parseString( - str, nimState.identCache, nimState.config, errorHandler = handleError + str, gState.identCache, gState.config, errorHandler = handleError ) except: decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 2ce4989..047a909 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -30,28 +30,12 @@ import "."/[globals, getters, comphelp, tshelp] # for where Nim can't (such as uint + -int) type - ExprParser* = ref object - state*: NimState - code*: string - name*: string - ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = - ExprParser(state: state, code: code, name: name) - -template techo(msg: varargs[string, `$`]) = - block: - let nimState {.inject.} = exprParser.state - decho join(msg, "") - template val(node: TSNode): string = - exprParser.code.getNodeVal(node) + gState.currentExpr.getNodeVal(node) -proc mode(exprParser: ExprParser): string = - exprParser.state.gState.mode - -proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank @@ -59,19 +43,19 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent var ident = identName if ident != "_": # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, kind, parent) + ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: - result = exprParser.state.getIdent(ident) - elif ident.nBl and ident in exprParser.state.constIdentifiers: - if exprParser.name.nBl: - ident = ident & "." & exprParser.name - result = exprParser.state.getIdent(ident) + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) -proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank - exprParser.getIdent(node.val, kind, parent) + gState.getExprIdent(node.val, kind, parent) proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed @@ -125,53 +109,60 @@ proc getCharLit(charStr: string): PNode {.inline.} = result = newNode(nkCharLit) result.intVal = parseChar(charStr).int64 +proc getFloatNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim float node from a C float expression + suffix + let floatSuffix = number[number.len-1] + try: + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + result = newFloatNode(nkFloatLit, parseFloat(number)) + except ValueError: + raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + +proc getIntNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim int node from a C integer expression + suffix + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) + + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode - result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[number.len-1] - try: - case floatSuffix - of 'l', 'L': - # TODO: handle long double (128 bits) - # result = newNode(nkFloat128Lit) - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - of 'f', 'F': - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - else: - result = newFloatNode(nkFloatLit, parseFloat(number)) - except ValueError: - raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + getFloatNode(number, suffix) else: - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) - else: - result = newNode(nkIntLit) + getIntNode(number, suffix) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) - -proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -189,17 +180,17 @@ proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( - exprParser.state.getIdent("-"), + gState.getIdent("-"), result ) else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processCharacterLiteral(gState: State, node: TSNode): PNode = let val = node.val result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processStringLiteral(gState: State, node: TSNode): PNode = let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -215,9 +206,9 @@ proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = result = newStrNode(nkStrLit, nimStr) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -227,24 +218,24 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va case shiftSym of "<<": - result.add exprParser.state.getIdent("shl") + result.add gState.getIdent("shl") of ">>": - result.add exprParser.state.getIdent("shr") + result.add gState.getIdent("shr") else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let leftNode = exprParser.processTSNode(left, typeofNode) + let leftNode = gState.processTSNode(left, typeofNode) # If the typeofNode is nil, set it # to be the leftNode because C's type coercion # happens left to right, and we want to emulate it if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -252,15 +243,15 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va rightNode ) -proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i], typeofNode)) + result.add(gState.processTSNode(node[i], typeofNode)) -proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( - exprParser.processTSNode(node[0], typeofNode), - exprParser.processTSNode(node[1], typeofNode) + gState.processTSNode(node[0], typeofNode), + gState.processTSNode(node[1], typeofNode) ) proc getNimUnarySym(csymbol: string): string = @@ -292,7 +283,7 @@ proc getNimBinarySym(csymbol: string): string = else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") -proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) result = newNode(nkInfix) @@ -302,16 +293,16 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v binarySym = node.tsNodeChild(1).val.strip() nimSym = getNimBinarySym(binarySym) - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) + result.add gState.getIdent(nimSym) + let leftNode = gState.processTSNode(left, typeofNode) if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -319,7 +310,7 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let @@ -333,31 +324,31 @@ proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: va # Might be bad because we are overwriting the type # There's probably a better way of doing this if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") + typeofNode = gState.getIdent("int64") result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), + gState.getIdent(unarySym), nkPar.newTree( nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) + gState.getIdent("int64"), + gState.processTSNode(child, typeofNode) ) ) ) else: result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) + gState.getIdent(nimSym), + gState.processTSNode(child, typeofNode) ) -proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(exprParser, node, typeofNode) + let binExpr = processBinaryExpression(gState, node, typeofNode) # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, @@ -366,37 +357,37 @@ proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeof elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = processUnaryExpression(exprParser, node, typeofNode) + result = processUnaryExpression(gState, node, typeofNode) else: raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") -proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( - exprParser.state.getIdent("sizeof"), - exprParser.processTSNode(node[0], typeofNode) + gState.getIdent("sizeof"), + gState.processTSNode(node[0], typeofNode) ) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() - techo "NODE: ", nodeName, ", VAL: ", node.val + decho "NODE: ", nodeName, ", VAL: ", node.val case nodeName of "number_literal": # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 - result = exprParser.processNumberLiteral(node) + result = gState.processNumberLiteral(node) of "string_literal": # Input -> "foo\0\x42" # Output -> "foo\0" - result = exprParser.processStringLiteral(node) + result = gState.processStringLiteral(node) of "char_literal": # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences # Output -> - result = exprParser.processCharacterLiteral(node) + result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain # an ERROR node. If that's the case, they usually contain children with @@ -405,21 +396,21 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ if node.len == 1: - result = exprParser.processTSNode(node[0], typeofNode) + result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: result = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add exprParser.processTSNode(node[i], typeofNode) + result.add gState.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": # Input -> (IDENT - OTHERIDENT) # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) - result = exprParser.processParenthesizedExpr(node, typeofNode) + result = gState.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": # Input -> sizeof(char) # Output -> sizeof(cchar) - result = exprParser.processSizeofExpression(node, typeofNode) + result = gState.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", @@ -434,55 +425,61 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) - result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) + result = gState.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": # Input -> a >> b, a << b # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = exprParser.processShiftExpression(node, typeofNode) + result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) - result = exprParser.processCastExpression(node, typeofNode) + result = gState.processCastExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": # Input -> true, false # Output -> true, false - result = exprParser.state.parseString(node.val) + result = gState.parseString(node.val) of "type_descriptor", "sized_type_specifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc let ty = getType(node.val) if ty.len > 0: # If ty is not empty, one of C's builtin types has been found - result = exprParser.getIdent(ty, nskType, parent=node.getName()) + result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: - result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + result = gState.getExprIdent(node.val, nskType, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) - result = exprParser.getIdent(node, parent=node.getName()) + result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RESULT: ", result + decho "NODE RESULT: ", result -proc parseCExpression*(state: NimState, code: string, name = ""): PNode = +proc parseCExpression*(gState: State, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil - let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser.code, exprParser.mode): - result = exprParser.processTSNode(root, tnode) + withCodeAst(gState.currentExpr, gState.mode): + result = gState.processTSNode(root, tnode) except ExprParseError as e: - techo e.msg + decho e.msg result = newNode(nkNone) except Exception as e: - techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + decho "UNEXPECTED EXCEPTION: ", e.msg + result = newNode(nkNone) + + # Clear the state + gState.currentExpr = "" + gState.currentTyCastName = "" \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index b964da9..0d0b4cd 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,4 +1,4 @@ -import sequtils, sets, tables +import sequtils, sets, tables, strutils import regex @@ -93,6 +93,9 @@ type currentHeader*, impShort*, sourceFile*: string + # Used for the exprparser.nim module + currentExpr*, currentTyCastName*: string + data*: seq[tuple[name, val: string]] nodeBranch*: seq[string] @@ -119,6 +122,6 @@ when not declared(CIMPORT): else: gState.outputHandle.writeLine(args) - template decho*(str: untyped): untyped = + template decho*(args: varargs[string, `$`]): untyped = if gState.debug: - gecho str.getCommented() + gecho join(args, "").getCommented() \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index afb511d..98045bf 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -15,44 +15,16 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) -<<<<<<< HEAD - doAssert gState.code.nBl, "Empty file or preprocessor error" - - if gState.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif gState.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {gState.mode}" - - var - tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32) - root = tree.tsTreeRootNode() - - defer: - tree.tsTreeDelete() - - if gState.past: - gecho gState.printLisp(root) - elif gState.pnim: - if Feature.ast2 in gState.feature: - ast2.parseNim(gState, path, root) - else: - ast.parseNim(gState, path, root, astTable) - elif gState.preprocess: - gecho gState.code -======= withCodeAst(gState.code, gState.mode): if gState.past: gecho gState.printLisp(root) elif gState.pnim: if Feature.ast2 in gState.feature: - ast2.printNim(gState, path, root) + ast2.parseNim(gState, path, root) else: - ast.printNim(gState, path, root, astTable) + ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code ->>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( From 619421ba85389439374dea2e9201553c7a267ff6 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:15:40 -0600 Subject: [PATCH 29/44] Update code with comments and add debug expr proc --- nimterop/exprparser.nim | 143 +++++++++++++++++++++++++++++++++++++++- nimterop/getters.nim | 8 +-- 2 files changed, 144 insertions(+), 7 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 047a909..7acb375 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -35,6 +35,11 @@ type template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) +proc printDebugExpr*(gState: State, node: TSNode) = + if gState.debug: + gecho ("Input => " & node.val).getCommented() + gecho gState.currentExpr.printLisp(node).getCommented() + proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## @@ -187,10 +192,29 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral(gState: State, node: TSNode): PNode = + # Input => 'G' + # + # (char_literal 1 1 3 "'G'") + # + # Output => 'G' + # + # nkCharLit("G") let val = node.val result = getCharLit(val[1 ..< val.len - 1]) proc processStringLiteral(gState: State, node: TSNode): PNode = + # Input => "\n\rfoobar\0\'" + # + # (string_literal 1 1 16 ""\n\rfoobar\0\'"" + # (escape_sequence 1 2 2 "\n") + # (escape_sequence 1 4 2 "\r") + # (escape_sequence 1 12 2 "\0") + # (escape_sequence 1 14 2 "\'") + # ) + # + # Output => "\n\cfoobar\x00\'" + # + # nkStrLit("\x0A\x0Dfoobar\x00\'") let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -209,6 +233,26 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => a >> b + # + # (shift_expression 1 2 6 + # (identifier 1 2 1 "a") + # (identifier 1 7 1 "b") + # ) + # + # Output => a shr typeof(a)(b) + # + # nkInfix( + # nkIdent("shr"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let left = node[0] @@ -244,11 +288,56 @@ proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (a + b) + # + # (parenthesized_expression 1 1 7 + # (math_expression 1 2 5 + # (identifier 1 2 1 "a") + # (identifier 1 6 1 "b") + # ) + # ) + # + # Output => (typeof(a)(a + typeof(a)(b))) + # + # nkPar( + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkInfix( + # nkIdent("+"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) + # ) + # ) result = newNode(nkPar) for i in 0 ..< node.len(): result.add(gState.processTSNode(node[i], typeofNode)) proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (int)a + # + # (cast_expression 1 1 6 "(int)a" + # (type_descriptor 1 2 3 "int" + # (primitive_type 1 2 3 "int") + # ) + # (identifier 1 6 1 "a") + # ) + # + # Output => cast[cint](a) + # + # nkCast( + # nkIdent("cint"), + # nkIdent("a") + # ) result = nkCast.newTree( gState.processTSNode(node[0], typeofNode), gState.processTSNode(node[1], typeofNode) @@ -285,6 +374,27 @@ proc getNimBinarySym(csymbol: string): string = proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) + # + # Input => a == b + # + # (equality_expression 1 1 6 + # (identifier 1 1 1 "a") + # (identifier 1 6 1 "b") + # ) + # + # Output => a == typeof(a)(b) + # + # nkInfix( + # nkIdent("=="), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let @@ -311,6 +421,20 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => !a + # + # (logical_expression 1 1 2 "!a" + # (identifier 1 2 1 "a") + # ) + # + # Output => (not a) + # + # nkPar( + # nkPrefix( + # nkIdent("not"), + # nkIdent("a") + # ) + # ) result = newNode(nkPar) let @@ -342,6 +466,7 @@ proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) @@ -362,6 +487,20 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => sizeof(int) + # + # (sizeof_expression 1 1 11 "sizeof(int)" + # (type_descriptor 1 8 3 "int" + # (primitive_type 1 8 3 "int") + # ) + # ) + # + # Output => sizeof(cint) + # + # nkCall( + # nkIdent("sizeof"), + # nkIdent("cint") + # ) result = nkCall.newTree( gState.getIdent("sizeof"), gState.processTSNode(node[0], typeofNode) @@ -385,8 +524,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Output -> "foo\0" result = gState.processStringLiteral(node) of "char_literal": - # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences - # Output -> + # Input -> 'F', '\060' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> 'F', '0', 'Z', '\r' result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b7744ce..121c8d5 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -436,13 +436,13 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: - result &= "'" & pnode.intVal.char & "')" + result &= ($pnode.intVal.char).escape & ")" of nkIntLit..nkUInt64Lit: result &= $pnode.intVal & ")" of nkFloatLit..nkFloat128Lit: result &= $pnode.floatVal & ")" of nkStrLit..nkTripleStrLit: - result &= "\"" & pnode.strVal & "\")" + result &= pnode.strVal.escape & ")" of nkSym: result &= $pnode.sym & ")" of nkIdent: @@ -460,16 +460,14 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(gState: State, node: TSNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug: gecho ("Input => " & gState.getNodeVal(node)).getCommented() gecho gState.printLisp(node).getCommented() proc printDebug*(gState: State, pnode: PNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug and pnode.kind != nkNone: gecho ("Output => " & $pnode).getCommented() - gecho gState.printTree(pnode).getCommented() + gecho gState.printTree(pnode) # Compiler shortcuts From fafdc42876212232e9c66c9fadff1507793cc2cd Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:45:19 -0600 Subject: [PATCH 30/44] Consolidate shift_expression into binary_expression and write more tests --- nimterop/exprparser.nim | 91 +++++++++-------------------------------- tests/include/tast2.h | 10 +++++ tests/tast2.nim | 14 ++++++- 3 files changed, 42 insertions(+), 73 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 7acb375..a59a546 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -232,61 +232,6 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = - # Input => a >> b - # - # (shift_expression 1 2 6 - # (identifier 1 2 1 "a") - # (identifier 1 7 1 "b") - # ) - # - # Output => a shr typeof(a)(b) - # - # nkInfix( - # nkIdent("shr"), - # nkIdent("a"), - # nkCall( - # nkCall( - # nkIdent("typeof"), - # nkIdent("a") - # ), - # nkIdent("b") - # ) - # ) - result = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let shiftSym = node.tsNodeChild(1).val.strip() - - case shiftSym - of "<<": - result.add gState.getIdent("shl") - of ">>": - result.add gState.getIdent("shr") - else: - raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - - let leftNode = gState.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - gState.getIdent("typeof"), - leftNode - ) - - let rightNode = gState.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => (a + b) # @@ -369,6 +314,10 @@ proc getNimBinarySym(csymbol: string): string = result = csymbol of "%": result = "mod" + of "<<": + result = "shl" + of ">>": + result = "shr" else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") @@ -419,6 +368,15 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) typeofNode, rightNode ) + if binarySym == "/": + # Special case. Nim's operators generally output + # the same type they take in, except for division. + # So we need to emulate C here and cast the whole + # expression to the type of the first arg + result = nkCall.newTree( + typeofNode, + result + ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => !a @@ -469,17 +427,7 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) - - # Make sure the statement is of the same type as the left - # hand argument, since some expressions return a differing - # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(gState, node, typeofNode) - # Note that this temp var binExpr is needed for some reason, or else we get a segfault - result = nkCall.newTree( - typeofNode, - binexpr - ) - + result = processBinaryExpression(gState, node, typeofNode) elif node.len() == 1: # Node has only one child, ie -(20 + 7) result = processUnaryExpression(gState, node, typeofNode) @@ -553,8 +501,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", - "bitwise_expression", "equality_expression", "binary_expression": - # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + "bitwise_expression", "equality_expression", "binary_expression", + "shift_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b # Output -> # typeof(a)(a == typeof(a)(b)) # typeof(a)(a != typeof(a)(b)) @@ -564,11 +513,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) + # a shr typeof(a)(b) + # a shl typeof(a)(b) result = gState.processUnaryOrBinaryExpression(node, typeofNode) - of "shift_expression": - # Input -> a >> b, a << b - # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 53b4d64..e72a3b8 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,6 +24,16 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 +#define a 100 +#define b 200 +#define EQ1 a <= b +#define EQ2 a >= b +#define EQ3 a > b +#define EQ4 a < b +#define EQ5 a != b +#define EQ6 a == b + +#define SIZEOF sizeof(char) #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index e82430b..57bfc90 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,6 +115,18 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) +assert a == 100 +assert b == 200 + +assert EQ1 == (a <= b) +assert EQ2 == (a >= b) +assert EQ3 == (a > b) +assert EQ4 == (a < b) +assert EQ5 == (a != b) +assert EQ6 == (a == b) + +assert SIZEOF == 1 + assert COERCE == 645635670332'u64 assert COERCE2 == 645635670332'i64 @@ -303,7 +315,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[123 + type(123)(132), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 1bf7e3ddb8ab03618a6151868f4686f178b39c6a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:53:52 -0600 Subject: [PATCH 31/44] Modify tmath.nim to have one skip symbol --- tests/tmath.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/tmath.nim b/tests/tmath.nim index bbf7abd..5a234db 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,10 +13,7 @@ when defined(windows): complex = object static: - when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] - else: - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] + cSkipSymbol @["math_errhandling"] cDebug() cDisableCaching() cAddStdDir() From 372ae5cef1512ca487ce0d35720d161873e9ce78 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:56:43 -0600 Subject: [PATCH 32/44] Fix macro expansion with common value --- tests/include/tast2.h | 16 ++++++++-------- tests/tast2.nim | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e72a3b8..815791f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,14 +24,14 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 -#define a 100 -#define b 200 -#define EQ1 a <= b -#define EQ2 a >= b -#define EQ3 a > b -#define EQ4 a < b -#define EQ5 a != b -#define EQ6 a == b +#define AVAL 100 +#define BVAL 200 +#define EQ1 AVAL <= BVAL +#define EQ2 AVAL >= BVAL +#define EQ3 AVAL > BVAL +#define EQ4 AVAL < BVAL +#define EQ5 AVAL != BVAL +#define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) diff --git a/tests/tast2.nim b/tests/tast2.nim index 57bfc90..da7fafa 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,15 +115,15 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert a == 100 -assert b == 200 +assert AVAL == 100 +assert BVAL == 200 -assert EQ1 == (a <= b) -assert EQ2 == (a >= b) -assert EQ3 == (a > b) -assert EQ4 == (a < b) -assert EQ5 == (a != b) -assert EQ6 == (a == b) +assert EQ1 == (AVAL <= BVAL) +assert EQ2 == (AVAL >= BVAL) +assert EQ3 == (AVAL > BVAL) +assert EQ4 == (AVAL < BVAL) +assert EQ5 == (AVAL != BVAL) +assert EQ6 == (AVAL == BVAL) assert SIZEOF == 1 From cb4a9d844fe547703606a6e646aaf05c396807c5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 17:07:06 -0600 Subject: [PATCH 33/44] Fix tmath again --- tests/tmath.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tmath.nim b/tests/tmath.nim index 5a234db..b8477c1 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,7 +13,12 @@ when defined(windows): complex = object static: - cSkipSymbol @["math_errhandling"] + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + # FP_ILOGB0 and FP_ILOGBNAN are casts that are unsupported + # on lower Nim VMs + cSkipSymbol @["math_errhandling", "FP_ILOGB0", "FP_ILOGBNAN"] + else: + cSkipSymbol @["math_errhandling"] cDebug() cDisableCaching() cAddStdDir() From 9ef3182d7a2b6f42878f6f5e32c41597939da565 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 19:21:57 -0600 Subject: [PATCH 34/44] Add hack for skipping types as root nodes --- nimterop/ast2.nim | 30 ++++++++++++++++++++++++++---- nimterop/exprparser.nim | 21 +++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4fec237..bb41ebe 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,9 +4,9 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/api +import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, exprparser, comphelp] +import "."/[globals, getters, exprparser, comphelp, tshelp] proc getPtrType*(str: string): string = result = case str: @@ -99,8 +99,30 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) - valident = - gState.parseCExpression(val) + var valident = newNode(nkNone) + + withCodeAst(val, gState.mode): + # This section is a hack for determining that the first + # node is a type, which shouldn't be accepted by a const + # def section. Need to replace this with some other mechanism + # to handle type aliases + var maybeTyNode: TSNode + # Take the very first node, which may be 2 levels + # down if there is an error node + if root.len > 0 and root[0].getName() == "ERROR": + maybeTyNode = root[0][0] + elif root.len > 0: + maybeTyNode = root[0] + + if not maybeTyNode.isNil: + let name = maybeTyNode.getName() + case name + of "type_descriptor", "sized_type_specifier": + discard + else: + # Can't do gState.parseCExpression(root) here for some reason? + # get a SEGFAULT if we use root + valident = gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a59a546..6e102ea 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -547,18 +547,15 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = decho "NODE RESULT: ", result -proc parseCExpression*(gState: State, code: string, name = ""): PNode = - ## Convert the C string to a nim PNode tree - gState.currentExpr = code - gState.currentTyCastName = name +proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = + ## Parse a c expression from a root ts node - result = newNode(nkNone) - # This is used for keeping track of the type of the first + # This var is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil + result = newNode(nkNone) try: - withCodeAst(gState.currentExpr, gState.mode): - result = gState.processTSNode(root, tnode) + result = gState.processTSNode(codeRoot, tnode) except ExprParseError as e: decho e.msg result = newNode(nkNone) @@ -566,6 +563,14 @@ proc parseCExpression*(gState: State, code: string, name = ""): PNode = decho "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) +proc parseCExpression*(gState: State, code: string, name = ""): PNode = + ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + + withCodeAst(gState.currentExpr, gState.mode): + result = gState.parseCExpression(root, name) + # Clear the state gState.currentExpr = "" gState.currentTyCastName = "" \ No newline at end of file From eaecf9ebdfd8d65a63cf2e3d820ba023405dd249 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:20:05 -0600 Subject: [PATCH 35/44] Add skippedSymbols for determining if a type has been skipped --- nimterop/ast2.nim | 3 +++ nimterop/exprparser.nim | 25 +++++++++++++------------ nimterop/globals.nim | 3 +++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index bb41ebe..2ae4a70 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -41,6 +41,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS result = pnode[0][0] else: gecho &"\n# $1'{origname}' skipped" % skind + gState.skippedSyms.incl origname if gState.debug: gState.skipStr &= &"\n{gState.getNodeVal(node)}" @@ -99,6 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) + var valident = newNode(nkNone) withCodeAst(val, gState.mode): @@ -151,6 +153,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = gecho &"# const '{origname}' is duplicate, skipped" else: gecho &"# const '{origname}' has invalid value '{val}'" + gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = # Add a const to the AST diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 6e102ea..1f6decc 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,16 +45,17 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - var ident = identName - if ident != "_": - # Process the identifier through cPlugin - ident = gState.getIdentifier(ident, kind, parent) - if kind == nskType: - result = gState.getIdent(ident) - elif ident.nBl and ident in gState.constIdentifiers: - if gState.currentTyCastName.nBl: - ident = ident & "." & gState.currentTyCastName - result = gState.getIdent(ident) + if identName notin gState.skippedSyms: + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = gState.getIdentifier(ident, kind, parent) + if kind == nskType: + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` @@ -534,8 +535,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: result = gState.getExprIdent(node.val, nskType, parent=node.getName()) - if result.kind == nkNone: - raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 0d0b4cd..d433ab5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,6 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] + # All symbols that have been skipped + skippedSyms*: HashSet[string] + # Legacy ast fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From c55e77e547ed7bab509990cdb91316c502f585d2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:59:21 -0600 Subject: [PATCH 36/44] Fix comments breaking code --- nimterop/exprparser.nim | 14 ++++++++++++-- tests/include/tast2.h | 4 ++-- tests/tpcre.nim | 1 - 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f6decc..e9ee502 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,9 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - result = newNode(nkStmtListExpr) + let res = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add gState.processTSNode(node[i], typeofNode) + let node = gState.processTSNode(node[i], typeofNode) + if node.kind != nkNone: + res.add node + if res.len == 1: + result = res[0] + elif res.len > 1: + result = res + else: + result = newNode(nkNone) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": @@ -543,6 +551,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") + of "comment": + discard else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 815791f..42c852f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,8 +35,8 @@ extern "C" { #define SIZEOF sizeof(char) -#define NULLCHAR '\0' -#define OCTCHAR '\012' +#define NULLCHAR '\0'/* comments should not break things*/ +#define OCTCHAR '\012' // nor should this comment #define HEXCHAR '\xFE' #define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 3bbd1ba..c8e8059 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,7 +7,6 @@ const pcreH = baseDir/"pcre.h.in" static: - cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From 49136a541d111fe860cf18c209e375e8d96451b5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 21:52:34 -0600 Subject: [PATCH 37/44] Don't support multiple base nodes yet --- nimterop/exprparser.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e9ee502..c74f0b6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,17 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - let res = newNode(nkStmtListExpr) + var nodes: seq[PNode] for i in 0 ..< node.len: - let node = gState.processTSNode(node[i], typeofNode) - if node.kind != nkNone: - res.add node - if res.len == 1: - result = res[0] - elif res.len > 1: - result = res - else: - result = newNode(nkNone) + let subNode = gState.processTSNode(node[i], typeofNode) + if subNode.kind != nkNone: + nodes.add(subNode) + # Multiple nodes can get tricky. Don't support them yet, unless they + # have at most one valid node + if nodes.len > 1: + raise newException(ExprParseError, &"Node type \"{nodeName}\" with val ({node.val}) has more than one non empty node") + if nodes.len == 1: + result = nodes[0] else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": From d11fd7407dd0055f31d04bf2571e90180fcbbf7e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 22:11:39 -0600 Subject: [PATCH 38/44] Add test for not supported nodes --- tests/include/tast2.h | 2 ++ tests/tast2.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 42c852f..b47a801 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -34,6 +34,8 @@ extern "C" { #define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) +#define REG_STR "regular string" +#define NOTSUPPORTEDSTR "not a " REG_STR #define NULLCHAR '\0'/* comments should not break things*/ #define OCTCHAR '\012' // nor should this comment diff --git a/tests/tast2.nim b/tests/tast2.nim index da7fafa..4cfbeac 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -109,6 +109,8 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert not defined(NOTSUPPORTEDSTR) + assert UEXPR == (1234.uint shl 1) assert ULEXPR == (1234.uint32 shl 2) assert ULLEXPR == (1234.uint64 shl 3) From e059b867196059fefbb5a0a245db149e11c07448 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 14:25:42 -0600 Subject: [PATCH 39/44] Address some PR comments --- nimterop/ast2.nim | 12 +++++++----- nimterop/globals.nim | 4 +++- nimterop/tshelp.nim | 14 ++++++-------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2ae4a70..474ec69 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,7 +4,7 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/[api, c, cpp] +import "."/treesitter/api import "."/[globals, getters, exprparser, comphelp, tshelp] @@ -1391,7 +1391,9 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] - fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + # Hold all of field information so that we can add all of them + # after the const identifiers has been updated + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1410,9 +1412,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fvalSections.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]))) else: - fvalSections.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode))) fnames.incl fname prev = fname @@ -1422,7 +1424,7 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fvalSections: + for (fname, fval, cexprNode) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d433ab5..5db17a3 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,7 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] - # All symbols that have been skipped + # All symbols that have been skipped due to + # being unwrappable or the user provided + # override is blank skippedSyms*: HashSet[string] # Legacy ast fields, remove when ast2 becomes default diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index f234bc0..109321c 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,11 +1,9 @@ -template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code +import "."/treesitter/[c, cpp] - # This section is needed to be able to reference - # mode in strformat calls - let - code = inputCode - mode {.inject.} = inputMode +template withCodeAst*(code: string, mode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + mixin treeSitterC + mixin treeSitterCpp var parser = tsParserNew() defer: @@ -18,7 +16,7 @@ template withCodeAst*(inputCode: string, inputMode: string, body: untyped): unty elif mode == "cpp": doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" else: - doAssert false, &"Invalid parser {mode}" + doAssert false, "Invalid parser " & mode var tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) From 35be85ad5662dd24dcaff0b6be1a05cbac77923f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 20:42:30 -0600 Subject: [PATCH 40/44] Add comments for enums and procs --- nimterop/ast2.nim | 36 +++++++++++++++++++++++++----------- nimterop/getters.nim | 21 ++++++++++++++++++++- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 474ec69..dbab355 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1393,14 +1393,18 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] if en.getName() == "comment": continue + let - fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField) + atom = en.getAtom() + commentNode = en.getInlineCommentNode() + fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) + if fname.nBl: var fval = "" @@ -1412,9 +1416,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]), commentNode)) else: - fieldDeclarations.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode), commentNode)) fnames.incl fname prev = fname @@ -1424,18 +1428,20 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fieldDeclarations: + for (fname, fval, cexprNode, commentNode) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + constNode.comment = gState.getCommentVal(commentNode) + gState.constSection.add constNode # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode) = +proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = # Add a proc variable decho("addProcVar()") let @@ -1488,12 +1494,13 @@ proc addProcVar(gState: State, node, rnode: TSNode) = # nkEmpty() # ) + identDefs.comment = gState.getCommentVal(commentNode) # nkVarSection.add gState.varSection.add identDefs gState.printDebug(identDefs) -proc addProc(gState: State, node, rnode: TSNode) = +proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1599,6 +1606,8 @@ proc addProc(gState: State, node, rnode: TSNode) = procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) + procDef.comment = gState.getCommentVal(commentNode) + # nkProcSection.add gState.procSection.add procDef @@ -1609,18 +1618,20 @@ proc addDecl(gState: State, node: TSNode) = decho("addDecl()") gState.printDebug(node) + let start = getStartAtom(node) + commentNode = node.getCommentNode() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - gState.addProcVar(node[i], node[start]) + gState.addProcVar(node[i], node[start], commentNode) else: # proc - gState.addProc(node[i], node[start]) + gState.addProc(node[i], node[start], commentNode) else: # Regular var discard @@ -1632,11 +1643,14 @@ proc addDef(gState: State, node: TSNode) = # and will fail at link time decho("addDef()") gState.printDebug(node) + let start = getStartAtom(node) + commentNode = node.getCommentNode() + if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): - gState.addProc(node[start+1], node[start]) + gState.addProc(node[start+1], node[start], commentNode) else: gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % gState.getNodeVal(node[start+1].getAtom()) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 121c8d5..d78715c 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times +import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options import regex @@ -431,6 +431,9 @@ proc printLisp*(gState: State, root: TSNode): string = proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") +proc getDocStrCommented*(str: string): string = + "\n## " & str.strip().replace("\n", "\n## ") + proc printTree*(gState: State, pnode: PNode, offset = ""): string = if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" @@ -609,6 +612,22 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] +proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = + if commentNode.isSome(): + result = gState.getNodeVal(commentNode.get()).replace(re"( *(/\*\*|\*\*/|\*/|\*))", "").strip() + +proc getCommentNode*(node: TSNode): Option[TSNode] = + result = none(TSNode) + let prevSibling = node.tsNodePrevNamedSibling() + if not prevSibling.isNil and prevSibling.getName() == "comment": + result = some(prevSibling) + +proc getInlineCommentNode*(node: TSNode): Option[TSNode] = + result = none(TSNode) + let nextSibling = node.tsNodeNextNamedSibling() + if not nextSibling.isNil and nextSibling.getName() == "comment": + result = some(nextSibling) + proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: for i in 0 .. node.tsNodeNamedChildCount()-1: From e867497e0b9fd20797bff261ede8481a70d1075f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 22:50:14 -0600 Subject: [PATCH 41/44] Add comments for fields and objects --- nimterop/ast2.nim | 26 ++++++++++++++++++++++---- nimterop/getters.nim | 24 ++++++++++++++---------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dbab355..b16f411 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,6 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") + commentNode = node[i].getNextCommentNode() # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -716,6 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Add nkIdentDefs for each field for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: + field.comment = gState.getCommentVal(commentNode) result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = @@ -725,6 +727,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") + let commentNode = node.tsNodeParent().getPrevCommentNode() + let # Object has fields or not fdlist = node.anyChildInTree("field_declaration_list") @@ -837,6 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add + typeDef.comment = gState.getCommentVal(commentNode) gState.typeSection.add typeDef gState.printDebug(typeDef) @@ -848,6 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) + commentNode = node.getNextCommentNode() # Fix issue #185 name = @@ -859,6 +865,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" if name.nBl and gState.identifierNodes.hasKey(name): let def = gState.identifierNodes[name] + def.comment = gState.getCommentVal(commentNode) + # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and def[2].kind == nkObjectTy and def[2].len == 3 and @@ -890,6 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) + commentNode = node.getPrevCommentNode() for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -897,6 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: + typeDef.comment = gState.getCommentVal(commentNode) let name = typeDef.getIdentName() @@ -1386,7 +1396,15 @@ proc addEnum(gState: State, node: TSNode) = gState.typeSection.add eoverride elif gState.addNewIdentifer(name): # Add enum definition and helpers - gState.enumSection.add gState.parseString(&"defineEnum({name})") + let defineNode = gState.parseString(&"defineEnum({name})") + # nkStmtList( + # nkCall( + # nkIdent("defineEnum"), + # nkIdent(name) <- set the comment here + # ) + # ) + defineNode[0][1].comment = gState.getCommentVal(node.getPrevCommentNode()) + gState.enumSection.add defineNode # Create const for fields var @@ -1402,7 +1420,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNode = en.getInlineCommentNode() + commentNode = en.getNextCommentNode() fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl: @@ -1621,7 +1639,7 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getCommentNode() + commentNode = node.getPrevCommentNode() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: @@ -1646,7 +1664,7 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getCommentNode() + commentNode = node.getPrevCommentNode() if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): diff --git a/nimterop/getters.nim b/nimterop/getters.nim index d78715c..bae8b5b 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -614,19 +614,23 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = if commentNode.isSome(): - result = gState.getNodeVal(commentNode.get()).replace(re"( *(/\*\*|\*\*/|\*/|\*))", "").strip() + result = gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").strip() -proc getCommentNode*(node: TSNode): Option[TSNode] = +template findComment(procName: untyped): untyped = result = none(TSNode) - let prevSibling = node.tsNodePrevNamedSibling() - if not prevSibling.isNil and prevSibling.getName() == "comment": - result = some(prevSibling) + var sibling = node.`procName`() + var i = 0 + while not sibling.isNil and i < maxSearch: + if sibling.getName() == "comment": + return some(sibling) + sibling = sibling.`procName`() + i += 1 -proc getInlineCommentNode*(node: TSNode): Option[TSNode] = - result = none(TSNode) - let nextSibling = node.tsNodeNextNamedSibling() - if not nextSibling.isNil and nextSibling.getName() == "comment": - result = some(nextSibling) +proc getPrevCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = + findComment(tsNodePrevNamedSibling) + +proc getNextCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = + findComment(tsNodeNextNamedSibling) proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: From eba4d3bc6b691251fd4965690e6c7c77baf8c937 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 24 Apr 2020 23:41:56 -0600 Subject: [PATCH 42/44] Fix comments not being valid rst --- nimterop/getters.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index bae8b5b..707817c 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -614,7 +614,7 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = if commentNode.isSome(): - result = gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").strip() + result = "::\n " & gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").replace("\n", "\n ").strip() template findComment(procName: untyped): untyped = result = none(TSNode) @@ -626,10 +626,10 @@ template findComment(procName: untyped): untyped = sibling = sibling.`procName`() i += 1 -proc getPrevCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = +proc getPrevCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = findComment(tsNodePrevNamedSibling) -proc getNextCommentNode*(node: TSNode, maxSearch=4): Option[TSNode] = +proc getNextCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = findComment(tsNodeNextNamedSibling) proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = From 3497c5732b72400e716531ce7598aa84ade83fb7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 13:08:52 -0600 Subject: [PATCH 43/44] Add ability to have multiple comments --- nimterop/ast2.nim | 49 ++++++++++++++++++++++---------------------- nimterop/getters.nim | 40 ++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index b16f411..e4eb38b 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,7 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") - commentNode = node[i].getNextCommentNode() + commentNodes = node[i].getNextCommentNodes() # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -717,7 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Add nkIdentDefs for each field for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: - field.comment = gState.getCommentVal(commentNode) + field.comment = gState.getCommentsStr(commentNodes) result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = @@ -727,7 +727,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") - let commentNode = node.tsNodeParent().getPrevCommentNode() + let commentNodes = node.tsNodeParent().getPrevCommentNodes() let # Object has fields or not @@ -841,7 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add - typeDef.comment = gState.getCommentVal(commentNode) + typeDef.comment = gState.getCommentsStr(commentNodes) gState.typeSection.add typeDef gState.printDebug(typeDef) @@ -853,7 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) - commentNode = node.getNextCommentNode() + commentNodes = node.getNextCommentNodes() # Fix issue #185 name = @@ -865,7 +865,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" if name.nBl and gState.identifierNodes.hasKey(name): let def = gState.identifierNodes[name] - def.comment = gState.getCommentVal(commentNode) + def.comment = gState.getCommentsStr(commentNodes) # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and @@ -898,7 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -906,7 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: - typeDef.comment = gState.getCommentVal(commentNode) + typeDef.comment = gState.getCommentsStr(commentNodes) let name = typeDef.getIdentName() @@ -1403,7 +1403,7 @@ proc addEnum(gState: State, node: TSNode) = # nkIdent(name) <- set the comment here # ) # ) - defineNode[0][1].comment = gState.getCommentVal(node.getPrevCommentNode()) + defineNode[0][1].comment = gState.getCommentsStr(node.getPrevCommentNodes()) gState.enumSection.add defineNode # Create const for fields @@ -1411,7 +1411,7 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: Option[TSNode]]] + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1420,7 +1420,7 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() - commentNode = en.getNextCommentNode() + commentNodes = en.getNextCommentNodes() fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) if fname.nBl: @@ -1434,9 +1434,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]), commentNode)) + fieldDeclarations.add((fname, "", some(en[1]), commentNodes)) else: - fieldDeclarations.add((fname, fval, none(TSNode), commentNode)) + fieldDeclarations.add((fname, fval, none(TSNode), commentNodes)) fnames.incl fname prev = fname @@ -1446,20 +1446,20 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode, commentNode) in fieldDeclarations: + for (fname, fval, cexprNode, commentNodes) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] - constNode.comment = gState.getCommentVal(commentNode) + constNode.comment = gState.getCommentsStr(commentNodes) gState.constSection.add constNode # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = +proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc variable decho("addProcVar()") let @@ -1512,13 +1512,13 @@ proc addProcVar(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) # nkEmpty() # ) - identDefs.comment = gState.getCommentVal(commentNode) + identDefs.comment = gState.getCommentsStr(commentNodes) # nkVarSection.add gState.varSection.add identDefs gState.printDebug(identDefs) -proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = +proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1624,7 +1624,7 @@ proc addProc(gState: State, node, rnode: TSNode, commentNode: Option[TSNode]) = procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) - procDef.comment = gState.getCommentVal(commentNode) + procDef.comment = gState.getCommentsStr(commentNodes) # nkProcSection.add gState.procSection.add procDef @@ -1636,20 +1636,19 @@ proc addDecl(gState: State, node: TSNode) = decho("addDecl()") gState.printDebug(node) - let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - gState.addProcVar(node[i], node[start], commentNode) + gState.addProcVar(node[i], node[start], commentNodes) else: # proc - gState.addProc(node[i], node[start], commentNode) + gState.addProc(node[i], node[start], commentNodes) else: # Regular var discard @@ -1664,11 +1663,11 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) - commentNode = node.getPrevCommentNode() + commentNodes = node.getPrevCommentNodes() if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): - gState.addProc(node[start+1], node[start], commentNode) + gState.addProc(node[start+1], node[start], commentNodes) else: gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % gState.getNodeVal(node[start+1].getAtom()) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 707817c..a181d09 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,5 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options +import algorithm import regex @@ -612,12 +613,13 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] -proc getCommentVal*(gState: State, commentNode: Option[TSNode]): string = - if commentNode.isSome(): - result = "::\n " & gState.getNodeVal(commentNode.get()).replace(re" *(/\*\*|\*\*/|\*/|\*)", "").replace("\n", "\n ").strip() +proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = + if commentNodes.len > 0: + result = "::" + for commentNode in commentNodes: + result &= "\n " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() template findComment(procName: untyped): untyped = - result = none(TSNode) var sibling = node.`procName`() var i = 0 while not sibling.isNil and i < maxSearch: @@ -626,11 +628,33 @@ template findComment(procName: untyped): untyped = sibling = sibling.`procName`() i += 1 -proc getPrevCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = - findComment(tsNodePrevNamedSibling) +proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = + ## Here we want to go until the node we get is not a comment + ## for cases with multiple ``//`` comments instead of one ``/* */`` + ## section + var sibling = node.tsNodePrevNamedSibling() + var i = 0 + while not sibling.isNil and i < maxSearch: + while not sibling.isNil and sibling.getName() == "comment": + result.add(sibling) + sibling = sibling.tsNodePrevNamedSibling() + if sibling.isNil: + result.reverse + return + sibling = sibling.tsNodePrevNamedSibling() + i += 1 -proc getNextCommentNode*(node: TSNode, maxSearch=1): Option[TSNode] = - findComment(tsNodeNextNamedSibling) + result.reverse + +proc getNextCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = + ## We only want to search for the next comment node (ie: inline) + var sibling = node.tsNodeNextNamedSibling() + var i = 0 + while not sibling.isNil and i < maxSearch: + if sibling.getName() == "comment": + return @[sibling] + sibling = sibling.tsNodeNextNamedSibling() + i += 1 proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: From 60fa8c40d1e0ef4b41e7c47aba406befd00c11a6 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 20:47:28 -0600 Subject: [PATCH 44/44] Remove unnecessary proc --- nimterop/getters.nim | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index a181d09..cadf83b 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times, options +import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import algorithm import regex @@ -619,15 +619,6 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n ").strip() -template findComment(procName: untyped): untyped = - var sibling = node.`procName`() - var i = 0 - while not sibling.isNil and i < maxSearch: - if sibling.getName() == "comment": - return some(sibling) - sibling = sibling.`procName`() - i += 1 - proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] = ## Here we want to go until the node we get is not a comment ## for cases with multiple ``//`` comments instead of one ``/* */``