From 80018f43cf35214e95c4fa777f65bb756be1db24 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 14 Apr 2020 20:38:35 -0600 Subject: [PATCH 01/24] WIP Ast printing --- nimterop/ast2.nim | 30 +++++++++++++++++++++++++++++- nimterop/getters.nim | 21 +++++++++++++++------ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index ce45f1d..43b6b35 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,10 +4,35 @@ import regex import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] -import "."/treesitter/api +import "."/treesitter/[api, c, cpp] import "."/[globals, getters] +proc getCCodeAst*(gState: State, code: string): string = + var parser = tsParserNew() + var code = code + + defer: + parser.tsParserDelete() + + + doAssert code.nBl, "Empty code" + if gState.mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif gState.mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {gState.mode}" + + var + tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) + root = tree.tsTreeRootNode() + + defer: + tree.tsTreeDelete() + + return code.printLisp(root) + proc getPtrType*(str: string): string = result = case str: of "cchar": @@ -59,6 +84,9 @@ proc getLit*(gState: State, str: string, expression = false): PNode = result = newStrNode(nkStrLit, str[1 .. ^2]) else: + decho "Macro AST:" + decho str + decho nimState.gState.getCCodeAst(str) let str = if expression: gState.getNimExpression(str) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 2d8d9bb..4ffb5d7 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -228,9 +228,12 @@ proc getName*(node: TSNode): string {.inline.} = if not node.isNil: return $node.tsNodeType() -proc getNodeVal*(gState: State, node: TSNode): string = +proc getNodeVal*(code: var string, node: TSNode): string = if not node.isNil: - return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + +proc getNodeVal*(gState: State, node: TSNode): string = + gState.code.getNodeVal(node) proc getAtom*(node: TSNode): TSNode = if not node.isNil: @@ -349,13 +352,16 @@ proc inChildren*(node: TSNode, ntype: string): bool = result = true break -proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = +proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = # Get line number and column info for node let point = node.tsNodeStartPoint() result.line = point.row.int + 1 result.col = point.column.int + 1 +proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getLineCol(gState.code, node) + proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = for i in 0 ..< node.len: if node.getName() != "comment": @@ -374,7 +380,7 @@ proc getPxName*(node: TSNode, offset: int): string = if count == offset and not np.isNil: return np.getName() -proc printLisp*(gState: State, root: TSNode): string = +proc printLisp*(code: var string, root: TSNode): string = var node = root nextnode: TSNode @@ -384,10 +390,10 @@ proc printLisp*(gState: State, root: TSNode): string = if not node.isNil and depth > -1: result &= spaces(depth) let - (line, col) = gState.getLineCol(node) + (line, col) = code.getLineCol(node) result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" let - val = gState.getNodeVal(node) + val = code.getNodeVal(node) if "\n" notin val and " " notin val: result &= &" \"{val}\"" else: @@ -419,6 +425,9 @@ proc printLisp*(gState: State, root: TSNode): string = if node == root: break +proc printLisp*(gState: State, root: TSNode): string = + printLisp(gState.code, root) + proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") From 879c7e3c784b3debce068a93e7f9ee116eb7bc21 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 16 Apr 2020 20:49:06 -0600 Subject: [PATCH 02/24] Add preliminary expression parsing --- nimterop/ast2.nim | 77 ++----------- nimterop/exprparser.nim | 245 ++++++++++++++++++++++++++++++++++++++++ nimterop/getters.nim | 2 +- 3 files changed, 253 insertions(+), 71 deletions(-) create mode 100644 nimterop/exprparser.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 43b6b35..04bb8eb 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,37 +1,10 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import regex - import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] -import "."/treesitter/[api, c, cpp] +import "."/treesitter/api -import "."/[globals, getters] - -proc getCCodeAst*(gState: State, code: string): string = - var parser = tsParserNew() - var code = code - - defer: - parser.tsParserDelete() - - - doAssert code.nBl, "Empty code" - if gState.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif gState.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {gState.mode}" - - var - tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) - root = tree.tsTreeRootNode() - - defer: - tree.tsTreeDelete() - - return code.printLisp(root) +import "."/[globals, getters, exprparser] proc getPtrType*(str: string): string = result = case str: @@ -59,42 +32,8 @@ proc parseString(gState: State, str: string): PNode = except: decho getCurrentExceptionMsg() -proc getLit*(gState: State, str: string, expression = false): PNode = - # Used to convert #define literals into const and expressions - # in array sizes - # - # `expression` is true when `str` should be converted into a Nim expression - let - str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - - if str.contains(re"^[\-]?[\d]+$"): # decimal - result = newIntNode(nkIntLit, parseInt(str)) - - elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float - result = newFloatNode(nkFloatLit, parseFloat(str)) - - elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal - result = gState.parseString(str) - - elif str.contains(re"^'[[:ascii:]]'$"): # char - result = newNode(nkCharLit) - result.intVal = str[1].int64 - - elif str.contains(re"""^"[[:ascii:]]+"$"""): # char * - result = newStrNode(nkStrLit, str[1 .. ^2]) - - else: - decho "Macro AST:" - decho str - decho nimState.gState.getCCodeAst(str) - let - str = - if expression: gState.getNimExpression(str) - else: str - result = gState.parseString(str) - - if result.isNil: - result = newNode(nkNilLit) +proc getLit*(nimState: NimState, str: string, expression = false): PNode = + result = nimState.codeToNode(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -181,11 +120,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank - result = gState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind in {nkCharLit .. nkStrLit} or - (valident.kind == nkStmtList and valident.len > 0 and - valident[0].kind in {nkCharLit .. nkStrLit}): - if gState.addNewIdentifer(name): + result = nimState.getOverrideOrSkip(node, origname, nskConst) + elif valident.kind != nkNilLit: + if nimState.addNewIdentifer(name): # const X* = Y # # nkConstDef( diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim new file mode 100644 index 0000000..a4be5d0 --- /dev/null +++ b/nimterop/exprparser.nim @@ -0,0 +1,245 @@ +import strformat, strutils, macros + +import regex + +import compiler/[ast, renderer] + +import "."/treesitter/[api, c, cpp] + +import "."/[globals, getters] + +type + ExprParser* = ref object + state*: NimState + code*: string + +proc newExprParser*(state: NimState, code: string): ExprParser = + ExprParser(state: state, code: code) + +template decho(msg: varargs[string, `$`]) = + if exprParser.state.gState.debug: + let nimState {.inject.} = exprParser.state + necho "# " & join(msg, "") + +template val*(node: TSNode): string = + exprParser.code.getNodeVal(node) + +proc mode*(exprParser: ExprParser): string = + exprParser.state.gState.mode + +template withCodeAst(exprParser: ExprParser, body: untyped): untyped = + var parser = tsParserNew() + defer: + parser.tsParserDelete() + + doAssert exprParser.code.nBl, "Empty code" + if exprParser.mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif exprParser.mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {exprParser.mode}" + + var + tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32) + root {.inject.} = tree.tsTreeRootNode() + + body + + defer: + tree.tsTreeDelete() + + +proc getNumNode(number, suffix: string): PNode {.inline.} = + result = newNode(nkNilLit) + if number.contains("."): + let floatSuffix = number[result.len-1] + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + discard + return + + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) + + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) + +proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkNilLit) + let nodeVal = node.val + + var match: RegexMatch + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)" + let found = nodeVal.find(reg, match) + if found: + let + prefix = if match.group(0).len > 0: nodeVal[match.group(0)[0]] else: "" + number = nodeVal[match.group(1)[0]] + suffix = nodeVal[match.group(2)[0]] + + result = getNumNode(number, suffix) + + if result.kind != nkNilLit and prefix == "-": + result = nkPrefix.newTree( + exprParser.state.getIdent("-"), + result + ) + +proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkCharLit) + result.intVal = node.val[1].int64 + +proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = + let nodeVal = node.val + result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) + +proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode + +proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkInfix) + let + left = node[0] + right = node[1] + var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + + case shiftSym + of "<<": + result.add exprParser.state.getIdent("shl") + of ">>": + result.add exprParser.state.getIdent("shr") + else: + discard + + result.add exprParser.processTSNode(left) + result.add exprParser.processTSNode(right) + +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkPar) + for i in 0 ..< node.len(): + result.add(exprParser.processTSNode(node[i])) + +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + decho "LOG SYM: ", binarySym + + case binarySym + of "!": + nimSym = "not" + else: + return newNode(nkNilLit) + + decho "LOG CHILD: ", child.val, ", nim: ", nimSym + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child) + ) + +proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = + if node.len() > 1: + result = newNode(nkInfix) + let left = node[0] + let right = node[1] + var nimSym = "" + + var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + decho "# BIN SYM: ", binarySym + + case binarySym + of "|", "||": + nimSym = "or" + of "&", "&&": + nimSym = "and" + of "^": + nimSym = "xor" + else: + return newNode(nkNilLit) + + result.add exprParser.state.getIdent(nimSym) + result.add exprParser.processTSNode(left) + result.add exprParser.processTSNode(right) + + elif node.len() == 1: + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + decho "# BIN SYM: ", binarySym + + case binarySym + of "~": + nimSym = "not" + else: + return newNode(nkNilLit) + + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child) + ) + +proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = + result = newNode(nkNilLit) + decho "# NODE: ", node.getName(), ", VAL: ", node.val + case node.getName() + of "number_literal": + result = exprParser.processNumberLiteral(node) + of "string_literal": + result = exprParser.processStringLiteral(node) + of "char_literal": + result = exprParser.processCharacterLiteral(node) + of "expression_statement", "ERROR", "translation_unit": + # This may be wrong. What can be in an expression? + result = exprParser.processTSNode(node[0]) + of "parenthesized_expression": + result = exprParser.processParenthesizedExpr(node) + of "bitwise_expression": + result = exprParser.processBitwiseExpression(node) + of "shift_expression": + result = exprParser.processShiftExpression(node) + of "logical_expression": + result = exprParser.processLogicalExpression(node) + of "identifier": + var ident = node.val + if ident != "_": + ident = exprParser.state.getIdentifier(ident, nskConst) + result = exprParser.state.getIdent(ident) + else: + result = newNode(nkNilLit) + + decho "# NODERES: ", result + +proc codeToNode*(state: NimState, code: string): PNode = + let exprParser = newExprParser(state, code) + withCodeAst(exprParser): + result = exprParser.processTSNode(root) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 4ffb5d7..b96ae70 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -221,7 +221,7 @@ proc len*(node: TSNode): int = result = node.tsNodeNamedChildCount().int proc `[]`*(node: TSNode, i: SomeInteger): TSNode = - if i < node.len: + if i < type(i)(node.len()): result = node.tsNodeNamedChild(i.uint32) proc getName*(node: TSNode): string {.inline.} = From c1520b4fd103da78c15c807092f7bcbb46d16012 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 06:33:02 -0600 Subject: [PATCH 03/24] Add better error handling for unimplemented types --- nimterop/exprparser.nim | 75 ++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a4be5d0..9570788 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -13,13 +13,15 @@ type state*: NimState code*: string + ExprParseError* = object of CatchableError + proc newExprParser*(state: NimState, code: string): ExprParser = ExprParser(state: state, code: code) -template decho(msg: varargs[string, `$`]) = +template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: let nimState {.inject.} = exprParser.state - necho "# " & join(msg, "") + necho "# " & join(msg, "").replace("\n", "\n# ") template val*(node: TSNode): string = exprParser.code.getNodeVal(node) @@ -54,16 +56,19 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result = newNode(nkNilLit) if number.contains("."): let floatSuffix = number[result.len-1] - case floatSuffix - of 'l', 'L': - # TODO: handle long double (128 bits) - # result = newNode(nkFloat128Lit) - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - of 'f', 'F': - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - else: - discard - return + try: + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1])) + return + except ValueError: + raise newException(ExprParseError, &"Could not parse float value \"{number}\".") case suffix of "u", "U": @@ -111,6 +116,8 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = exprParser.state.getIdent("-"), result ) + else: + raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkCharLit) @@ -135,7 +142,7 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = of ">>": result.add exprParser.state.getIdent("shr") else: - discard + raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") result.add exprParser.processTSNode(left) result.add exprParser.processTSNode(right) @@ -151,15 +158,15 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - decho "LOG SYM: ", binarySym + techo "LOG SYM: ", binarySym case binarySym of "!": nimSym = "not" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"") - decho "LOG CHILD: ", child.val, ", nim: ", nimSym + techo "LOG CHILD: ", child.val, ", nim: ", nimSym result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), exprParser.processTSNode(child) @@ -173,7 +180,7 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() - decho "# BIN SYM: ", binarySym + techo "BIN SYM: ", binarySym case binarySym of "|", "||": @@ -183,7 +190,7 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = of "^": nimSym = "xor" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) result.add exprParser.processTSNode(left) @@ -195,23 +202,26 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = var nimSym = "" var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - decho "# BIN SYM: ", binarySym + techo "BIN SYM: ", binarySym case binarySym of "~": nimSym = "not" else: - return newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"") result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), exprParser.processTSNode(child) ) + else: + raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkNilLit) - decho "# NODE: ", node.getName(), ", VAL: ", node.val - case node.getName() + let nodeName = node.getName() + techo "NODE: ", nodeName, ", VAL: ", node.val + case nodeName of "number_literal": result = exprParser.processNumberLiteral(node) of "string_literal": @@ -220,7 +230,11 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = result = exprParser.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # This may be wrong. What can be in an expression? - result = exprParser.processTSNode(node[0]) + if node.len > 0: + result = exprParser.processTSNode(node[0]) + else: + raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") + of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node) of "bitwise_expression": @@ -235,11 +249,18 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = ident = exprParser.state.getIdentifier(ident, nskConst) result = exprParser.state.getIdent(ident) else: - result = newNode(nkNilLit) + raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - decho "# NODERES: ", result + techo "NODERES: ", result proc codeToNode*(state: NimState, code: string): PNode = let exprParser = newExprParser(state, code) - withCodeAst(exprParser): - result = exprParser.processTSNode(root) \ No newline at end of file + try: + withCodeAst(exprParser): + result = exprParser.processTSNode(root) + except ExprParseError as e: + techo e.msg + result = newNode(nkNilLit) + except Exception as e: + techo e.msg + result = newNode(nkNilLit) \ No newline at end of file From 305d90583e5dc7a8dde012c982047df9dee21f8e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 06:53:41 -0600 Subject: [PATCH 04/24] Add proper binary casting --- nimterop/exprparser.nim | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 9570788..1f72a9a 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -144,8 +144,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - result.add exprParser.processTSNode(left) - result.add exprParser.processTSNode(right) + let + leftNode = exprParser.processTSNode(left) + rightNode = exprParser.processTSNode(right) + + result.add leftNode + result.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ), + rightNode + ) proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = result = newNode(nkPar) @@ -193,8 +203,18 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) - result.add exprParser.processTSNode(left) - result.add exprParser.processTSNode(right) + let + leftNode = exprParser.processTSNode(left) + rightNode = exprParser.processTSNode(right) + + result.add leftNode + result.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ), + rightNode + ) elif node.len() == 1: result = newNode(nkPar) From 055d6bee73eb988ae498a57d29dd3d80a8928721 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 17 Apr 2020 07:37:17 -0600 Subject: [PATCH 05/24] Add math_expression and fix ast2 tests Update to include more expressions --- nimterop/ast2.nim | 24 +---- nimterop/exprparser.nim | 200 ++++++++++++++++++++++++++++++---------- nimterop/getters.nim | 22 +++-- nimterop/globals.nim | 2 +- tests/include/tast2.h | 18 ++++ tests/tast2.nim | 22 ++++- 6 files changed, 211 insertions(+), 77 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 04bb8eb..85b555f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,10 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer] +import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser] +import "."/[globals, getters, exprparser, utils] proc getPtrType*(str: string): string = result = case str: @@ -17,21 +17,6 @@ proc getPtrType*(str: string): string = else: str -proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = - # Raise exception in parseString() instead of exiting for errors - if msg < warnMin: - raise newException(Exception, msgKindToString(msg)) - -proc parseString(gState: State, str: string): PNode = - # Parse a string into Nim AST - use custom error handler that raises - # an exception rather than exiting on failure - try: - result = parseString( - str, gState.identCache, gState.config, errorHandler = handleError - ) - except: - decho getCurrentExceptionMsg() - proc getLit*(nimState: NimState, str: string, expression = false): PNode = result = nimState.codeToNode(str) @@ -121,7 +106,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank result = nimState.getOverrideOrSkip(node, origname, nskConst) - elif valident.kind != nkNilLit: + elif valident.kind != nkNone: if nimState.addNewIdentifer(name): # const X* = Y # @@ -179,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - let + var colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1501,6 +1486,7 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") + var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f72a9a..505453c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters] +import "."/[globals, getters, utils] type ExprParser* = ref object @@ -21,15 +21,16 @@ proc newExprParser*(state: NimState, code: string): ExprParser = template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: let nimState {.inject.} = exprParser.state - necho "# " & join(msg, "").replace("\n", "\n# ") + necho join(msg, "").getCommented -template val*(node: TSNode): string = +template val(node: TSNode): string = exprParser.code.getNodeVal(node) -proc mode*(exprParser: ExprParser): string = +proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode template withCodeAst(exprParser: ExprParser, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() defer: parser.tsParserDelete() @@ -51,9 +52,9 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() - proc getNumNode(number, suffix: string): PNode {.inline.} = - result = newNode(nkNilLit) + ## Convert a C number to a Nim number PNode + result = newNode(nkNone) if number.contains("."): let floatSuffix = number[result.len-1] try: @@ -84,6 +85,8 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result = newNode(nkIntLit) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. if number.contains(re"0[xX]"): result.intVal = parseHexInt(number) result.flags = {nfBase16} @@ -97,11 +100,11 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) + result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -111,7 +114,7 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = result = getNumNode(number, suffix) - if result.kind != nkNilLit and prefix == "-": + if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( exprParser.state.getIdent("-"), result @@ -127,9 +130,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkInfix) let left = node[0] @@ -144,25 +147,29 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode result.add nkCast.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i])) + result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -179,14 +186,96 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode = techo "LOG CHILD: ", child.val, ", nim: ", nimSym result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) -proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + if node.len > 1: + # Node has left and right children ie: (2 + 7) + var + res = newNode(nkInfix) + let + left = node[0] + right = node[1] + + let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + techo "MATH SYM: ", mathSym + + res.add exprParser.state.getIdent(mathSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) + + res.add leftNode + # res.add rightNode + res.add nkCast.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + rightNode + ) + + # Make sure the statement is of the same type as the left + # hand argument, since some expressions return a differing + # type than the input types (2/3 == float) + result = nkCall.newTree( + nkCall.newTree( + exprParser.state.getIdent("typeof"), + tnode + ), + res + ) + + elif node.len() == 1: + # Node has only one child, ie -(20 + 7) + result = newNode(nkPar) + let child = node[0] + var nimSym = "" + + let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "MATH SYM: ", unarySym + + case unarySym + of "+": + nimSym = "+" + of "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gental cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + return + else: + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + else: + raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = if node.len() > 1: result = newNode(nkInfix) - let left = node[0] - let right = node[1] + + let + left = node[0] + right = node[1] + var nimSym = "" var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() @@ -203,15 +292,19 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) - let - leftNode = exprParser.processTSNode(left) - rightNode = exprParser.processTSNode(right) + let leftNode = exprParser.processTSNode(left, typeofNode) + + var tnode = typeofNode + if tnode.isNil: + tnode = leftNode + + let rightNode = exprParser.processTSNode(right, tnode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( nkCall.newTree( exprParser.state.getIdent("typeof"), - leftNode + tnode ), rightNode ) @@ -221,24 +314,26 @@ proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode = let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() - techo "BIN SYM: ", binarySym + var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + techo "BIN SYM: ", unarySym - case binarySym + case unarySym of "~": nimSym = "not" else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") result.add nkPrefix.newTree( exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child) + exprParser.processTSNode(child, typeofNode) ) else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkNilLit) +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = + ## Handle all of the types of expressions here. This proc gets called recursively + ## in the processX procs and will drill down to sub nodes. + result = newNode(nkNone) let nodeName = node.getName() techo "NODE: ", nodeName, ", VAL: ", node.val case nodeName @@ -251,36 +346,47 @@ proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode = of "expression_statement", "ERROR", "translation_unit": # This may be wrong. What can be in an expression? if node.len > 0: - result = exprParser.processTSNode(node[0]) + result = exprParser.processTSNode(node[0], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") - of "parenthesized_expression": - result = exprParser.processParenthesizedExpr(node) + result = exprParser.processParenthesizedExpr(node, typeofNode) of "bitwise_expression": - result = exprParser.processBitwiseExpression(node) + result = exprParser.processBitwiseExpression(node, typeofNode) + of "math_expression": + result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": - result = exprParser.processShiftExpression(node) + result = exprParser.processShiftExpression(node, typeofNode) of "logical_expression": - result = exprParser.processLogicalExpression(node) + result = exprParser.processLogicalExpression(node, typeofNode) + # Why are these node types named true/false? + of "true", "false": + result = exprParser.state.parseString(node.val) of "identifier": var ident = node.val if ident != "_": + # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, nskConst) - result = exprParser.state.getIdent(ident) + techo ident + if ident != "": + result = exprParser.state.getIdent(ident) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODERES: ", result + techo "NODE RES: ", result proc codeToNode*(state: NimState, code: string): PNode = - let exprParser = newExprParser(state, code) + ## Convert the C string to a nim PNode tree + result = newNode(nkNone) try: + let exprParser = newExprParser(state, code) withCodeAst(exprParser): result = exprParser.processTSNode(root) except ExprParseError as e: - techo e.msg - result = newNode(nkNilLit) + echo e.msg.getCommented + result = newNode(nkNone) except Exception as e: - techo e.msg - result = newNode(nkNilLit) \ No newline at end of file + echo e.msg.getCommented + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b96ae70..f43f199 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -399,7 +399,7 @@ proc printLisp*(code: var string, root: TSNode): string = else: break - if node.tsNodeNamedChildCount() != 0: + if node.len() != 0: result &= "\n" nextnode = node.tsNodeNamedChild(0) depth += 1 @@ -459,15 +459,19 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(gState: State, node: TSNode) = - if gState.debug: - gecho ("Input => " & gState.getNodeVal(node)).getCommented() & "\n" & - gState.printLisp(node).getCommented() +proc printDebug*(nimState: NimState, node: TSNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug: + necho ("Input => " & nimState.getNodeVal(node)).getCommented() + necho nimState.gState.printLisp(node).getCommented() -proc printDebug*(gState: State, pnode: PNode) = - if gState.debug: - gecho ("Output => " & $pnode).getCommented() & "\n" & - gState.printTree(pnode) +proc printDebug*(nimState: NimState, pnode: PNode) = + discard + # This causes random segfaults for some reason on macOS Catalina + if nimState.gState.debug and pnode.kind != nkNone: + necho ("Output => " & $pnode).getCommented() + necho nimState.printTree(pnode) # Compiler shortcuts diff --git a/nimterop/globals.nim b/nimterop/globals.nim index f159124..b964da9 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -113,7 +113,7 @@ when not declared(CIMPORT): export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl # Redirect output to file when required - template gecho*(args: string) {.dirty.} = + template gecho*(args: string) = if gState.outputHandle.isNil: echo args else: diff --git a/tests/include/tast2.h b/tests/include/tast2.h index bdf8823..7e15ac0 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -8,6 +8,24 @@ extern "C" { #define D "hello" #define E 'c' +#define UEXPR (1234u << 1) +#define ULEXPR (1234ul << 2) +#define ULLEXPR (1234ull << 3) +#define LEXPR (1234l << 4) +#define LLEXPR (1234ll << 5) + +#define SHL1 (1u << 1) +#define SHL2 (1u << 2) +#define SHL3 (1u << 3) +#define COERCE 645635634896ull + -35436 +#define COERCE2 645635634896 + -35436 +#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define BOOL true +#define MATHEXPR (1 + 2/3*20 - 100) +#define ANDEXPR (100 & 11000) + +#define ALLSHL (SHL1 | SHL2 | SHL3) + struct A0; struct A1 {}; typedef struct A2; diff --git a/tests/tast2.nim b/tests/tast2.nim index e13c4ac..d65e34a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -105,6 +105,26 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert UEXPR == (1234.uint shl 1) +assert ULEXPR == (1234.uint32 shl 2) +assert ULLEXPR == (1234.uint64 shl 3) +assert LEXPR == (1234.int32 shl 4) +assert LLEXPR == (1234.int64 shl 5) + +assert COERCE == 645635599460'u64 +assert COERCE2 == 645635599460'i64 + +assert BINEXPR == 5 +assert BOOL == true +assert MATHEXPR == -99 +assert ANDEXPR == 96 + +assert SHL1 == (1.uint shl 1) +assert SHL2 == (1.uint shl 2) +assert SHL3 == (1.uint shl 3) + +assert ALLSHL == (SHL1 or SHL2 or SHL3) + assert A0 is object testFields(A0, "f1!cint") checkPragmas(A0, pHeaderBy, istype = false) @@ -271,7 +291,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[123 + 132, ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 62c68d69ee0409558a846aae3a85cf1a29f9442b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 18 Apr 2020 13:13:33 -0600 Subject: [PATCH 06/24] Add sizeof expression parser, fix nkNone issue, reduce size of typeof expressions Cleanup, fix bugs --- nimterop/ast2.nim | 2 +- nimterop/exprparser.nim | 149 +++++++++++++++++++++++++--------------- nimterop/getters.nim | 4 +- 3 files changed, 97 insertions(+), 58 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 85b555f..8c5da8c 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -963,7 +963,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo let # Size of array could be a Nim expression size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) - if size.kind != nkNilLit: + if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] elif cnode.len == 1: diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 505453c..d27f23b 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -29,6 +29,24 @@ template val(node: TSNode): string = proc mode(exprParser: ExprParser): string = exprParser.state.gState.mode +proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + result = newNode(nkNone) + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = exprParser.state.getIdentifier(ident, kind, parent) + if ident != "": + result = exprParser.state.getIdent(ident) + +proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = + ## Gets a cPlugin transformed identifier from `identName` + ## + ## Returns PNode(nkNone) if the identifier is blank + exprParser.getIdent(node.val, kind, parent) + template withCodeAst(exprParser: ExprParser, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code var parser = tsParserNew() @@ -56,7 +74,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[result.len-1] + let floatSuffix = number[number.len-1] try: case floatSuffix of 'l', 'L': @@ -66,7 +84,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = of 'f', 'F': result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: - result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1])) + result = newFloatNode(nkFloatLit, parseFloat(number)) return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") @@ -100,11 +118,12 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result.intVal = parseInt(number) proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = + ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?)([ulUL]*)" + const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" let found = nodeVal.find(reg, match) if found: let @@ -130,9 +149,9 @@ proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = let nodeVal = node.val result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode +proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -149,27 +168,35 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: P let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCast.newTree( + exprParser.processTSNode(node[0], typeofNode), + exprParser.processTSNode(node[1], typeofNode) + ) + +proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -189,7 +216,7 @@ proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: exprParser.processTSNode(child, typeofNode) ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) var @@ -204,19 +231,20 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo res.add exprParser.state.getIdent(mathSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + # If the typeofNode is nil, set it + # to be the leftNode because C's type coercion + # happens left to right, and we want to emulate it + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - # res.add rightNode res.add nkCast.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -224,10 +252,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # hand argument, since some expressions return a differing # type than the input types (2/3 == float) result = nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, res ) @@ -248,6 +273,8 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo # so we have to make a gental cast here to coerce it to one. # Might be bad because we are overwriting the type # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") result.add nkPrefix.newTree( exprParser.state.getIdent(unarySym), nkPar.newTree( @@ -268,7 +295,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNo else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len() > 1: result = newNode(nkInfix) @@ -288,24 +315,25 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: nimSym = "and" of "^": nimSym = "xor" + of "==", "!=": + nimSym = binarySym else: raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") result.add exprParser.state.getIdent(nimSym) let leftNode = exprParser.processTSNode(left, typeofNode) - var tnode = typeofNode - if tnode.isNil: - tnode = leftNode + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) - let rightNode = exprParser.processTSNode(right, tnode) + let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( - nkCall.newTree( - exprParser.state.getIdent("typeof"), - tnode - ), + typeofNode, rightNode ) @@ -317,6 +345,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() techo "BIN SYM: ", unarySym + # TODO: Support more symbols here case unarySym of "~": nimSym = "not" @@ -330,7 +359,13 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: else: raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil): PNode = +proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = nkCall.newTree( + exprParser.state.getIdent("sizeof"), + exprParser.processTSNode(node[0], typeofNode) + ) + +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) @@ -351,42 +386,48 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: PNode = nil raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": result = exprParser.processParenthesizedExpr(node, typeofNode) - of "bitwise_expression": + of "sizeof_expression": + result = exprParser.processSizeofExpression(node, typeofNode) + of "bitwise_expression", "equality_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) of "shift_expression": result = exprParser.processShiftExpression(node, typeofNode) + of "cast_expression": + result = exprParser.processCastExpression(node, typeofNode) of "logical_expression": result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "identifier": - var ident = node.val - if ident != "_": - # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, nskConst) - techo ident - if ident != "": - result = exprParser.state.getIdent(ident) + of "type_descriptor": + let ty = getType(node.val) + result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{ident}\"") + result = exprParser.state.getIdent(ty) + of "identifier": + result = exprParser.getIdent(node, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RES: ", result + techo "NODE RESULT: ", result proc codeToNode*(state: NimState, code: string): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) + # This is used for keeping track of the type of the first + # symbol + var tnode: PNode = nil + let exprParser = newExprParser(state, code) try: - let exprParser = newExprParser(state, code) withCodeAst(exprParser): - result = exprParser.processTSNode(root) + result = exprParser.processTSNode(root, tnode) except ExprParseError as e: - echo e.msg.getCommented + techo e.msg result = newNode(nkNone) except Exception as e: - echo e.msg.getCommented + techo "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f43f199..216d2e6 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -460,18 +460,16 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(nimState: NimState, node: TSNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug: necho ("Input => " & nimState.getNodeVal(node)).getCommented() necho nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = - discard # This causes random segfaults for some reason on macOS Catalina if nimState.gState.debug and pnode.kind != nkNone: necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode) + necho nimState.printTree(pnode).getCommented() # Compiler shortcuts From 173e6d625c3ea96a95580303fedd011684635cc4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 19 Apr 2020 19:18:04 -0600 Subject: [PATCH 07/24] Add string and char support Update some comments Rename exprparser main proc Don't export parse procs Add missing utils module Try to fix array type test Try fix cast test error Disable cast test for now Revert back comment test. Have to figure out how to test without vm --- nimterop/ast2.nim | 6 +-- nimterop/exprparser.nim | 101 +++++++++++++++++++++++++++++++++------- nimterop/utils.nim | 18 +++++++ tests/include/tast2.h | 6 +++ tests/tast2.nim | 12 +++-- 5 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 nimterop/utils.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8c5da8c..08ea2d6 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -18,7 +18,7 @@ proc getPtrType*(str: string): string = str proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.codeToNode(str) + result = nimState.parseCExpression(str) proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined @@ -164,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value: if value.isNil: pragma.add pident else: - var + let colExpr = newNode(nkExprColonExpr) colExpr.add pident colExpr.add value @@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) = if en.len > 1 and en[1].getName() in gEnumVals: # Explicit value - fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index d27f23b..183bb18 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros +import strformat, strutils, macros, sets import regex @@ -12,11 +12,12 @@ type ExprParser* = ref object state*: NimState code*: string + name*: string ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string): ExprParser = - ExprParser(state: state, code: code) +proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = + ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = if exprParser.state.gState.debug: @@ -38,6 +39,8 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) + if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: + ident = ident & "." & exprParser.name if ident != "": result = exprParser.state.getIdent(ident) @@ -70,6 +73,58 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped = defer: tree.tsTreeDelete() +proc parseChar(charStr: string): uint8 {.inline.} = + ## Parses a character literal out of a string. This is needed + ## because treesitter gives unescaped characters when parsing + ## strings. + if charStr.len == 1: + return charStr[0].uint8 + + # Handle octal, hex, unicode? + if charStr.startsWith("\\x"): + result = parseHexInt(charStr.replace("\\x", "0x")).uint8 + elif charStr.len == 4: # Octal + result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8 + + if result == 0: + case charStr + of "\\0": + result = ord('\0') + of "\\a": + result = 0x07 + of "\\b": + result = 0x08 + of "\\e": + result = 0x1B + of "\\f": + result = 0x0C + of "\\n": + result = '\n'.uint8 + of "\\r": + result = 0x0D + of "\\t": + result = 0x09 + of "\\v": + result = 0x0B + of "\\\\": + result = 0x5C + of "\\'": + result = '\''.uint8 + of "\\\"": + result = '\"'.uint8 + of "\\?": + result = 0x3F + else: + discard + + if result > uint8.high: + result = uint8.high + +proc getCharLit(charStr: string): PNode {.inline.} = + ## Convert a character string into a proper Nim char lit node + result = newNode(nkCharLit) + result.intVal = parseChar(charStr).int64 + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode result = newNode(nkNone) @@ -117,7 +172,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = else: result.intVal = parseInt(number) -proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -141,17 +196,29 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode = else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode = - result = newNode(nkCharLit) - result.intVal = node.val[1].int64 +proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = + let val = node.val + result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode = - let nodeVal = node.val - result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1]) +proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = + let + nodeVal = node.val + strVal = nodeVal[1 ..< nodeVal.len - 1] -proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode + const + str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + reg = re(str) -proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + # Convert the c string escape sequences/etc to Nim chars + var nimStr = newStringOfCap(nodeVal.len) + for m in strVal.findAll(reg): + nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) + + result = newStrNode(nkStrLit, nimStr) + +proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode + +proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -185,18 +252,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): result.add(exprParser.processTSNode(node[i], typeofNode)) -proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let child = node[0] var nimSym = "" @@ -415,13 +482,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): techo "NODE RESULT: ", result -proc codeToNode*(state: NimState, code: string): PNode = +proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol var tnode: PNode = nil - let exprParser = newExprParser(state, code) + let exprParser = newExprParser(state, code, name) try: withCodeAst(exprParser): result = exprParser.processTSNode(root, tnode) diff --git a/nimterop/utils.nim b/nimterop/utils.nim new file mode 100644 index 0000000..025256a --- /dev/null +++ b/nimterop/utils.nim @@ -0,0 +1,18 @@ +import compiler/[ast, lineinfos, msgs, options, parser, renderer] + +import "."/[globals, getters] + +proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = + # Raise exception in parseString() instead of exiting for errors + if msg < warnMin: + raise newException(Exception, msgKindToString(msg)) + +proc parseString*(nimState: NimState, str: string): PNode = + # Parse a string into Nim AST - use custom error handler that raises + # an exception rather than exiting on failure + try: + result = parseString( + str, nimState.identCache, nimState.config, errorHandler = handleError + ) + except: + decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 7e15ac0..3c6148a 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -23,6 +23,12 @@ extern "C" { #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) +#define CASTEXPR (int) 34 + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" #define ALLSHL (SHL1 | SHL2 | SHL3) diff --git a/tests/tast2.nim b/tests/tast2.nim index d65e34a..d00552c 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -93,11 +93,11 @@ macro testFields(t: typed, fields: static[string] = "") = for i in 0 ..< rl.len: let name = ($rl[i][0]).strip(chars = {'*'}) - typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "") + typ = ($(rl[i][1].repr())).replace("\n", "").replace(" ", "").replace("typeof", "type") n = names.find(name) assert n != -1, $t & "." & name & " invalid" - assert types[n] == typ, - "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ + assert types[n].replace("typeof", "type") == typ, + "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ assert A == 2 assert B == 1.0 @@ -118,6 +118,12 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 +assert CASTEXPR == 34.chr + +assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" +assert NULLCHAR == '\0' +assert OCTCHAR == '\n' +assert HEXCHAR.int == 0xFE assert SHL1 == (1.uint shl 1) assert SHL2 == (1.uint shl 2) From aa1086fae37999180b485ced323d35acec7868a1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 09:08:08 -0600 Subject: [PATCH 08/24] Fix cast type only run tast2 on supported compilers Change cast to gentle type cast Re enable tests for Nim < 1.0.0 --- nimterop/exprparser.nim | 6 +++--- tests/include/tast2.h | 6 +++--- tests/tast2.nim | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 183bb18..ed287a3 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -247,7 +247,7 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let rightNode = exprParser.processTSNode(right, typeofNode) result.add leftNode - result.add nkCast.newTree( + result.add nkCall.newTree( typeofNode, rightNode ) @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCast.newTree( + result = nkCall.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) @@ -310,7 +310,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let rightNode = exprParser.processTSNode(right, typeofNode) res.add leftNode - res.add nkCast.newTree( + res.add nkCall.newTree( typeofNode, rightNode ) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 3c6148a..53b4d64 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -17,13 +17,13 @@ extern "C" { #define SHL1 (1u << 1) #define SHL2 (1u << 2) #define SHL3 (1u << 3) -#define COERCE 645635634896ull + -35436 -#define COERCE2 645635634896 + -35436 +#define COERCE 645635634896ull + 35436 +#define COERCE2 645635634896 + 35436ul #define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) #define BOOL true #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) -#define CASTEXPR (int) 34 +#define CASTEXPR (char) 34 #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index d00552c..d43ce9f 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -111,8 +111,8 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert COERCE == 645635599460'u64 -assert COERCE2 == 645635599460'i64 +assert COERCE == 645635670332'u64 +assert COERCE2 == 645635670332'i64 assert BINEXPR == 5 assert BOOL == true @@ -297,7 +297,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[typeof(123)(123 + cast[typeof(123)](132)), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 208098b3eb959e477a3625a8a48305ddaa1cec4f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 11:33:53 -0600 Subject: [PATCH 09/24] Make bitwise_expression forwards compatible Skip syms for tmath Skip more syms Add casting back for cast_expression. Disable cast test on Nim < 1.0.0 Skip more syms for cast expressions on Nim < 1.0.0 --- nimterop/exprparser.nim | 8 +++++--- tests/tast2.nim | 11 +++++++++-- tests/tmath.nim | 4 ++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index ed287a3..207d7fd 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -258,7 +258,7 @@ proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: result.add(exprParser.processTSNode(node[i], typeofNode)) proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = nkCall.newTree( + result = nkCast.newTree( exprParser.processTSNode(node[0], typeofNode), exprParser.processTSNode(node[1], typeofNode) ) @@ -455,7 +455,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": result = exprParser.processSizeofExpression(node, typeofNode) - of "bitwise_expression", "equality_expression": + # binary_expression from the new treesitter upgrade should work here + # once we upgrade + of "bitwise_expression", "equality_expression", "binary_expression": result = exprParser.processBitwiseExpression(node, typeofNode) of "math_expression": result = exprParser.processMathExpression(node, typeofNode) @@ -497,4 +499,4 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + result = newNode(nkNone) diff --git a/tests/tast2.nim b/tests/tast2.nim index d43ce9f..0b3762a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -3,11 +3,16 @@ import macros, os, sets, strutils import nimterop/[cimport] static: + # Skip casting on lower nim compilers because + # the VM does not support it + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["CASTEXPR"] cDebug() const path = currentSourcePath.parentDir() / "include" / "tast2.h" + when defined(HEADER): cDefine("HEADER") const @@ -118,7 +123,9 @@ assert BINEXPR == 5 assert BOOL == true assert MATHEXPR == -99 assert ANDEXPR == 96 -assert CASTEXPR == 34.chr + +when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0): + assert CASTEXPR == 34.chr assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar" assert NULLCHAR == '\0' @@ -453,4 +460,4 @@ checkPragmas(nested, pHeaderImpBy) when defined(HEADER): assert sitest1(5) == 10 - assert sitest1(10) == 20 \ No newline at end of file + assert sitest1(10) == 20 diff --git a/tests/tmath.nim b/tests/tmath.nim index 5d84700..bbf7abd 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -13,6 +13,10 @@ when defined(windows): complex = object static: + when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] + else: + cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] cDebug() cDisableCaching() cAddStdDir() From 1576127a6e1b5f429d3fb91cf924fc41e43346cf Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:09:08 -0600 Subject: [PATCH 10/24] Use tsNodeChild --- nimterop/exprparser.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 207d7fd..610c0cb 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -223,7 +223,8 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va let left = node[0] right = node[1] - var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + + let shiftSym = node.tsNodeChild(1).val.strip() case shiftSym of "<<": @@ -268,7 +269,7 @@ proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(0).val.strip() techo "LOG SYM: ", binarySym case binarySym @@ -292,7 +293,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var left = node[0] right = node[1] - let mathSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let mathSym = node.tsNodeChild(1).val.strip() techo "MATH SYM: ", mathSym res.add exprParser.state.getIdent(mathSym) @@ -329,7 +330,7 @@ proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var let child = node[0] var nimSym = "" - let unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "MATH SYM: ", unarySym case unarySym @@ -372,7 +373,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var nimSym = "" - var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip() + let binarySym = node.tsNodeChild(1).val.strip() techo "BIN SYM: ", binarySym case binarySym @@ -409,7 +410,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let child = node[0] var nimSym = "" - var unarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip() + let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym # TODO: Support more symbols here @@ -437,7 +438,9 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() + techo "NODE: ", nodeName, ", VAL: ", node.val + case nodeName of "number_literal": result = exprParser.processNumberLiteral(node) From a84adfe1d8db39e91f4e0e0af4cd81f02cee2fd6 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 17:55:05 -0600 Subject: [PATCH 11/24] Add comments describing capabilities --- nimterop/exprparser.nim | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 610c0cb..1fc72ab 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -8,6 +8,27 @@ import "."/treesitter/[api, c, cpp] import "."/[globals, getters, utils] +# This version of exprparser should be able to handle: +# +# All integers + integer like expressions (hex, octal, suffixes) +# All floating point expressions (except for C++'s hex floating point stuff) +# Strings and character literals, including C's escape characters (not sure if this is the same as C++'s escape characters or not) +# Math operators (+, -, /, *) +# Some Unary operators (-, !, ~). ++, --, and & are yet to be implemented +# Any identifiers +# C type descriptors (int, char, etc) +# Boolean values (true, false) +# Shift expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Math expressions (containing anything in this list) +# Sizeof expressions (containing anything in this list) +# Cast expressions (containing anything in this list) +# Parentheses expressions (containing anything in this list) +# Expressions containing other expressions +# +# In addition to the above, it should also handle most type coercions, except +# for where Nim can't (such as uint + -int) + type ExprParser* = ref object state*: NimState @@ -413,7 +434,7 @@ proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: let unarySym = node.tsNodeChild(0).val.strip() techo "BIN SYM: ", unarySym - # TODO: Support more symbols here + # TODO: Support more symbols here. ++, --, & case unarySym of "~": nimSym = "not" From d83884a3917cf96b7df209d47c7301919cce3967 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 18:04:12 -0600 Subject: [PATCH 12/24] Add missing sized type specifier for 'long int', etc --- nimterop/exprparser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1fc72ab..44e59d6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -494,7 +494,7 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Why are these node types named true/false? of "true", "false": result = exprParser.state.parseString(node.val) - of "type_descriptor": + of "type_descriptor", "sized_type_specifier": let ty = getType(node.val) result = exprParser.getIdent(ty, nskType, parent=node.getName()) if result.kind == nkNone: From 4794c076ffc7f46858dfebfdae47d8e3c47d1cf7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 20 Apr 2020 19:01:53 -0600 Subject: [PATCH 13/24] Fix pcre tests --- tests/tpcre.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tpcre.nim b/tests/tpcre.nim index c8e8059..3bbd1ba 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,6 +7,7 @@ const pcreH = baseDir/"pcre.h.in" static: + cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From b1a445c34dfff04d5c799b7c4ee0f7e806900e91 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 01:16:19 -0600 Subject: [PATCH 14/24] Update based on comments from review. Need to add more docs and reorg to use gstate --- nimterop/ast2.nim | 30 +- nimterop/{utils.nim => comphelp.nim} | 0 nimterop/exprparser.nim | 394 ++++++++++++--------------- nimterop/getters.nim | 20 +- nimterop/toast.nim | 20 +- nimterop/tshelp.nim | 30 ++ tests/tast2.nim | 1 - 7 files changed, 249 insertions(+), 246 deletions(-) rename nimterop/{utils.nim => comphelp.nim} (100%) create mode 100644 nimterop/tshelp.nim diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 08ea2d6..1b7fb3d 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,10 +1,12 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times +import options as opts + import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api -import "."/[globals, getters, exprparser, utils] +import "."/[globals, getters, exprparser, comphelp] proc getPtrType*(str: string): string = result = case str: @@ -17,9 +19,6 @@ proc getPtrType*(str: string): string = else: str -proc getLit*(nimState: NimState, str: string, expression = false): PNode = - result = nimState.parseCExpression(str) - proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined # and use that if present @@ -101,7 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gState.getNodeVal(node[1]) valident = - gState.getLit(val) + gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank @@ -962,7 +961,7 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo # type name[X] => array[X, type] let # Size of array could be a Nim expression - size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true) + size = gState.parseCExpression(gState.getNodeVal(cnode[1])) if size.kind != nkNone: result = gState.newArrayTree(cnode, result, size) cnode = cnode[0] @@ -1367,6 +1366,7 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] + fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1385,20 +1385,25 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - # Explicit value - fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name - - # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + fvalSections.add((fname, "", some(en[1]))) + else: + fvalSections.add((fname, fval, none(TSNode))) fnames.incl fname - prev = fname # Add fields to list of consts after processing enum so that we don't cast # enum field to itself gState.constIdentifiers.incl fnames + # parseCExpression requires all const identifiers to be present for the enum + for (fname, fval, cexprNode) in fvalSections: + var fval = fval + if cexprNode.isSome: + fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) @@ -1486,7 +1491,6 @@ proc addProc(gState: State, node, rnode: TSNode) = # Parameter list plist = node.anyChildInTree("parameter_list") - var procDef = newNode(nkProcDef) # proc X(a1: Y, a2: Z): P {.pragma.} diff --git a/nimterop/utils.nim b/nimterop/comphelp.nim similarity index 100% rename from nimterop/utils.nim rename to nimterop/comphelp.nim diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 44e59d6..2ce4989 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -6,7 +6,7 @@ import compiler/[ast, renderer] import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, utils] +import "."/[globals, getters, comphelp, tshelp] # This version of exprparser should be able to handle: # @@ -41,9 +41,9 @@ proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = ExprParser(state: state, code: code, name: name) template techo(msg: varargs[string, `$`]) = - if exprParser.state.gState.debug: + block: let nimState {.inject.} = exprParser.state - necho join(msg, "").getCommented + decho join(msg, "") template val(node: TSNode): string = exprParser.code.getNodeVal(node) @@ -60,9 +60,11 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent if ident != "_": # Process the identifier through cPlugin ident = exprParser.state.getIdentifier(ident, kind, parent) - if exprParser.name.nBl and ident in exprParser.state.constIdentifiers: - ident = ident & "." & exprParser.name - if ident != "": + if kind == nskType: + result = exprParser.state.getIdent(ident) + elif ident.nBl and ident in exprParser.state.constIdentifiers: + if exprParser.name.nBl: + ident = ident & "." & exprParser.name result = exprParser.state.getIdent(ident) proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = @@ -71,29 +73,6 @@ proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = "" ## Returns PNode(nkNone) if the identifier is blank exprParser.getIdent(node.val, kind, parent) -template withCodeAst(exprParser: ExprParser, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code - var parser = tsParserNew() - defer: - parser.tsParserDelete() - - doAssert exprParser.code.nBl, "Empty code" - if exprParser.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif exprParser.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {exprParser.mode}" - - var - tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32) - root {.inject.} = tree.tsTreeRootNode() - - body - - defer: - tree.tsTreeDelete() - proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed ## because treesitter gives unescaped characters when parsing @@ -161,37 +140,36 @@ proc getNumNode(number, suffix: string): PNode {.inline.} = result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) else: result = newFloatNode(nkFloatLit, parseFloat(number)) - return except ValueError: raise newException(ExprParseError, &"Could not parse float value \"{number}\".") - - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) else: - result = newNode(nkIntLit) + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc @@ -285,168 +263,112 @@ proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var exprParser.processTSNode(node[1], typeofNode) ) -proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let binarySym = node.tsNodeChild(0).val.strip() - techo "LOG SYM: ", binarySym - - case binarySym - of "!": - nimSym = "not" +proc getNimUnarySym(csymbol: string): string = + ## Get the Nim equivalent of a unary C symbol + ## + ## TODO: Add ++, --, + case csymbol + of "+", "-": + result = csymbol + of "~", "!": + result = "not" else: - raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"") + raise newException(ExprParseError, &"Unsupported unary symbol \"{csymbol}\"") - techo "LOG CHILD: ", child.val, ", nim: ", nimSym - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) +proc getNimBinarySym(csymbol: string): string = + case csymbol + of "|", "||": + result = "or" + of "&", "&&": + result = "and" + of "^": + result = "xor" + of "==", "!=", + "+", "-", "/", "*", + ">", "<", ">=", "<=": + result = csymbol + of "%": + result = "mod" + else: + raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") + +proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + # Node has left and right children ie: (2 + 7) + result = newNode(nkInfix) + + let + left = node[0] + right = node[1] + binarySym = node.tsNodeChild(1).val.strip() + nimSym = getNimBinarySym(binarySym) + + result.add exprParser.state.getIdent(nimSym) + let leftNode = exprParser.processTSNode(left, typeofNode) + + if typeofNode.isNil: + typeofNode = nkCall.newTree( + exprParser.state.getIdent("typeof"), + leftNode + ) + + let rightNode = exprParser.processTSNode(right, typeofNode) + + result.add leftNode + result.add nkCall.newTree( + typeofNode, + rightNode ) -proc processMathExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = + result = newNode(nkPar) + + let + child = node[0] + unarySym = node.tsNodeChild(0).val.strip() + nimSym = getNimUnarySym(unarySym) + + if nimSym == "-": + # Special case. The minus symbol must be in front of an integer, + # so we have to make a gentle cast here to coerce it to one. + # Might be bad because we are overwriting the type + # There's probably a better way of doing this + if typeofNode.isNil: + typeofNode = exprParser.state.getIdent("int64") + + result.add nkPrefix.newTree( + exprParser.state.getIdent(unarySym), + nkPar.newTree( + nkCall.newTree( + exprParser.state.getIdent("int64"), + exprParser.processTSNode(child, typeofNode) + ) + ) + ) + else: + result.add nkPrefix.newTree( + exprParser.state.getIdent(nimSym), + exprParser.processTSNode(child, typeofNode) + ) + +proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) - var - res = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let mathSym = node.tsNodeChild(1).val.strip() - techo "MATH SYM: ", mathSym - - res.add exprParser.state.getIdent(mathSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - res.add leftNode - res.add nkCall.newTree( - typeofNode, - rightNode - ) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) + let binExpr = processBinaryExpression(exprParser, node, typeofNode) + # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, - res + binexpr ) elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "MATH SYM: ", unarySym - - case unarySym - of "+": - nimSym = "+" - of "-": - # Special case. The minus symbol must be in front of an integer, - # so we have to make a gental cast here to coerce it to one. - # Might be bad because we are overwriting the type - # There's probably a better way of doing this - if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") - result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), - nkPar.newTree( - nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) - ) - ) - ) - return - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) + result = processUnaryExpression(exprParser, node, typeofNode) else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") - -proc processBitwiseExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = - if node.len() > 1: - result = newNode(nkInfix) - - let - left = node[0] - right = node[1] - - var nimSym = "" - - let binarySym = node.tsNodeChild(1).val.strip() - techo "BIN SYM: ", binarySym - - case binarySym - of "|", "||": - nimSym = "or" - of "&", "&&": - nimSym = "and" - of "^": - nimSym = "xor" - of "==", "!=": - nimSym = binarySym - else: - raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"") - - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) - - if typeofNode.isNil: - typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), - leftNode - ) - - let rightNode = exprParser.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - - elif node.len() == 1: - result = newNode(nkPar) - let child = node[0] - var nimSym = "" - - let unarySym = node.tsNodeChild(0).val.strip() - techo "BIN SYM: ", unarySym - - # TODO: Support more symbols here. ++, --, & - case unarySym - of "~": - nimSym = "not" - else: - raise newException(ExprParseError, &"Unsupported unary symbol \"{unarySym}\"") - - result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) - ) - else: - raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"") + raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( @@ -464,45 +386,85 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): case nodeName of "number_literal": + # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f + # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 result = exprParser.processNumberLiteral(node) of "string_literal": + # Input -> "foo\0\x42" + # Output -> "foo\0" result = exprParser.processStringLiteral(node) of "char_literal": + # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> result = exprParser.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": - # This may be wrong. What can be in an expression? - if node.len > 0: + # Note that we're parsing partial expressions, so the TSNode might contain + # an ERROR node. If that's the case, they usually contain children with + # partial results, which will contain parsed expressions + # + # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) + # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ + if node.len == 1: result = exprParser.processTSNode(node[0], typeofNode) + elif node.len > 1: + result = newNode(nkStmtListExpr) + for i in 0 ..< node.len: + result.add exprParser.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": + # Input -> (IDENT - OTHERIDENT) + # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) result = exprParser.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": + # Input -> sizeof(char) + # Output -> sizeof(cchar) result = exprParser.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade - of "bitwise_expression", "equality_expression", "binary_expression": - result = exprParser.processBitwiseExpression(node, typeofNode) - of "math_expression": - result = exprParser.processMathExpression(node, typeofNode) + of "math_expression", "logical_expression", "relational_expression", + "bitwise_expression", "equality_expression", "binary_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + # Output -> + # typeof(a)(a == typeof(a)(b)) + # typeof(a)(a != typeof(a)(b)) + # (not a) + # (not a) + # typeof(a)(a < typeof(a)(b)) + # typeof(a)(a > typeof(a)(b)) + # typeof(a)(a <= typeof(a)(b)) + # typeof(a)(a >= typeof(a)(b)) + result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": + # Input -> a >> b, a << b + # Output -> a shr typeof(a)(b), a shl typeof(a)(b) result = exprParser.processShiftExpression(node, typeofNode) of "cast_expression": + # Input -> (int) a + # Output -> cast[cint](a) result = exprParser.processCastExpression(node, typeofNode) - of "logical_expression": - result = exprParser.processLogicalExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": + # Input -> true, false + # Output -> true, false result = exprParser.state.parseString(node.val) of "type_descriptor", "sized_type_specifier": + # Input -> int, unsigned int, long int, etc + # Output -> cint, cuint, clong, etc let ty = getType(node.val) - result = exprParser.getIdent(ty, nskType, parent=node.getName()) - if result.kind == nkNone: - result = exprParser.state.getIdent(ty) + if ty.len > 0: + # If ty is not empty, one of C's builtin types has been found + result = exprParser.getIdent(ty, nskType, parent=node.getName()) + else: + result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": + # Input -> IDENT + # Output -> IDENT (if found in sym table, else error) result = exprParser.getIdent(node, parent=node.getName()) if result.kind == nkNone: - raise newException(ExprParseError, &"Could not get identifier \"{node.val}\"") + raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") @@ -512,15 +474,15 @@ proc parseCExpression*(state: NimState, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree result = newNode(nkNone) # This is used for keeping track of the type of the first - # symbol + # symbol used for type casting var tnode: PNode = nil let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser): + withCodeAst(exprParser.code, exprParser.mode): result = exprParser.processTSNode(root, tnode) except ExprParseError as e: techo e.msg result = newNode(nkNone) except Exception as e: techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) + result = newNode(nkNone) \ No newline at end of file diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 216d2e6..b7744ce 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -401,7 +401,7 @@ proc printLisp*(code: var string, root: TSNode): string = if node.len() != 0: result &= "\n" - nextnode = node.tsNodeNamedChild(0) + nextnode = node[0] depth += 1 else: result &= ")\n" @@ -432,7 +432,7 @@ proc getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") proc printTree*(gState: State, pnode: PNode, offset = ""): string = - if gState.debug and pnode.kind != nkNone: + if not pnode.isNil and gState.debug and pnode.kind != nkNone: result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: @@ -459,17 +459,17 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = if offset.len == 0: result &= "\n" -proc printDebug*(nimState: NimState, node: TSNode) = +proc printDebug*(gState: State, node: TSNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug: - necho ("Input => " & nimState.getNodeVal(node)).getCommented() - necho nimState.gState.printLisp(node).getCommented() + if gState.debug: + gecho ("Input => " & gState.getNodeVal(node)).getCommented() + gecho gState.printLisp(node).getCommented() -proc printDebug*(nimState: NimState, pnode: PNode) = +proc printDebug*(gState: State, pnode: PNode) = # This causes random segfaults for some reason on macOS Catalina - if nimState.gState.debug and pnode.kind != nkNone: - necho ("Output => " & $pnode).getCommented() - necho nimState.printTree(pnode).getCommented() + if gState.debug and pnode.kind != nkNone: + gecho ("Output => " & $pnode).getCommented() + gecho gState.printTree(pnode).getCommented() # Compiler shortcuts diff --git a/nimterop/toast.nim b/nimterop/toast.nim index ab44970..afb511d 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,16 +2,11 @@ import os, osproc, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, globals, getters, grammar, build] +import "."/[ast, ast2, globals, getters, grammar, build, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" - var parser = tsParserNew() - - defer: - parser.tsParserDelete() - if gState.mode.Bl: gState.mode = getCompilerMode(path) @@ -20,6 +15,7 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) +<<<<<<< HEAD doAssert gState.code.nBl, "Empty file or preprocessor error" if gState.mode == "c": @@ -45,6 +41,18 @@ proc process(gState: State, path: string, astTable: AstTable) = ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code +======= + withCodeAst(gState.code, gState.mode): + if gState.past: + gecho gState.printLisp(root) + elif gState.pnim: + if Feature.ast2 in gState.feature: + ast2.printNim(gState, path, root) + else: + ast.printNim(gState, path, root, astTable) + elif gState.preprocess: + gecho gState.code +>>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim new file mode 100644 index 0000000..f234bc0 --- /dev/null +++ b/nimterop/tshelp.nim @@ -0,0 +1,30 @@ +template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + + # This section is needed to be able to reference + # mode in strformat calls + let + code = inputCode + mode {.inject.} = inputMode + + var parser = tsParserNew() + defer: + parser.tsParserDelete() + + doAssert code.nBl, "Empty code or preprocessor error" + + if mode == "c": + doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" + elif mode == "cpp": + doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" + else: + doAssert false, &"Invalid parser {mode}" + + var + tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32) + root {.inject.} = tree.tsTreeRootNode() + + body + + defer: + tree.tsTreeDelete() \ No newline at end of file diff --git a/tests/tast2.nim b/tests/tast2.nim index 0b3762a..e82430b 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -12,7 +12,6 @@ static: const path = currentSourcePath.parentDir() / "include" / "tast2.h" - when defined(HEADER): cDefine("HEADER") const From 8a0f7954ae3754258706796ccd018f79ddb0158d Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 22 Apr 2020 20:20:34 -0600 Subject: [PATCH 15/24] nimState -> gState --- nimterop/ast2.nim | 8 +- nimterop/comphelp.nim | 4 +- nimterop/exprparser.nim | 247 ++++++++++++++++++++-------------------- nimterop/globals.nim | 9 +- nimterop/toast.nim | 32 +----- 5 files changed, 136 insertions(+), 164 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 1b7fb3d..4fec237 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -104,9 +104,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = if name.Bl: # Name skipped or overridden since blank - result = nimState.getOverrideOrSkip(node, origname, nskConst) + result = gState.getOverrideOrSkip(node, origname, nskConst) elif valident.kind != nkNone: - if nimState.addNewIdentifer(name): + if gState.addNewIdentifer(name): # const X* = Y # # nkConstDef( @@ -1400,9 +1400,9 @@ proc addEnum(gState: State, node: TSNode) = for (fname, fval, cexprNode) in fvalSections: var fval = fval if cexprNode.isSome: - fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name + fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0] + gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] # Add other names if node.getName() == "type_definition" and node.len > 1: diff --git a/nimterop/comphelp.nim b/nimterop/comphelp.nim index 025256a..1709f8b 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/comphelp.nim @@ -7,12 +7,12 @@ proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) if msg < warnMin: raise newException(Exception, msgKindToString(msg)) -proc parseString*(nimState: NimState, str: string): PNode = +proc parseString*(gState: State, str: string): PNode = # Parse a string into Nim AST - use custom error handler that raises # an exception rather than exiting on failure try: result = parseString( - str, nimState.identCache, nimState.config, errorHandler = handleError + str, gState.identCache, gState.config, errorHandler = handleError ) except: decho getCurrentExceptionMsg() \ No newline at end of file diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 2ce4989..047a909 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -30,28 +30,12 @@ import "."/[globals, getters, comphelp, tshelp] # for where Nim can't (such as uint + -int) type - ExprParser* = ref object - state*: NimState - code*: string - name*: string - ExprParseError* = object of CatchableError -proc newExprParser*(state: NimState, code: string, name = ""): ExprParser = - ExprParser(state: state, code: code, name: name) - -template techo(msg: varargs[string, `$`]) = - block: - let nimState {.inject.} = exprParser.state - decho join(msg, "") - template val(node: TSNode): string = - exprParser.code.getNodeVal(node) + gState.currentExpr.getNodeVal(node) -proc mode(exprParser: ExprParser): string = - exprParser.state.gState.mode - -proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank @@ -59,19 +43,19 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent var ident = identName if ident != "_": # Process the identifier through cPlugin - ident = exprParser.state.getIdentifier(ident, kind, parent) + ident = gState.getIdentifier(ident, kind, parent) if kind == nskType: - result = exprParser.state.getIdent(ident) - elif ident.nBl and ident in exprParser.state.constIdentifiers: - if exprParser.name.nBl: - ident = ident & "." & exprParser.name - result = exprParser.state.getIdent(ident) + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) -proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode = +proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## ## Returns PNode(nkNone) if the identifier is blank - exprParser.getIdent(node.val, kind, parent) + gState.getExprIdent(node.val, kind, parent) proc parseChar(charStr: string): uint8 {.inline.} = ## Parses a character literal out of a string. This is needed @@ -125,53 +109,60 @@ proc getCharLit(charStr: string): PNode {.inline.} = result = newNode(nkCharLit) result.intVal = parseChar(charStr).int64 +proc getFloatNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim float node from a C float expression + suffix + let floatSuffix = number[number.len-1] + try: + case floatSuffix + of 'l', 'L': + # TODO: handle long double (128 bits) + # result = newNode(nkFloat128Lit) + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + of 'f', 'F': + result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) + else: + result = newFloatNode(nkFloatLit, parseFloat(number)) + except ValueError: + raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + +proc getIntNode(number, suffix: string): PNode {.inline.} = + ## Get a Nim int node from a C integer expression + suffix + case suffix + of "u", "U": + result = newNode(nkUintLit) + of "l", "L": + result = newNode(nkInt32Lit) + of "ul", "UL": + result = newNode(nkUint32Lit) + of "ll", "LL": + result = newNode(nkInt64Lit) + of "ull", "ULL": + result = newNode(nkUint64Lit) + else: + result = newNode(nkIntLit) + + # I realize these regex are wasteful on performance, but + # couldn't come up with a better idea. + if number.contains(re"0[xX]"): + result.intVal = parseHexInt(number) + result.flags = {nfBase16} + elif number.contains(re"0[bB]"): + result.intVal = parseBinInt(number) + result.flags = {nfBase2} + elif number.contains(re"0[oO]"): + result.intVal = parseOctInt(number) + result.flags = {nfBase8} + else: + result.intVal = parseInt(number) + proc getNumNode(number, suffix: string): PNode {.inline.} = ## Convert a C number to a Nim number PNode - result = newNode(nkNone) if number.contains("."): - let floatSuffix = number[number.len-1] - try: - case floatSuffix - of 'l', 'L': - # TODO: handle long double (128 bits) - # result = newNode(nkFloat128Lit) - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - of 'f', 'F': - result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1])) - else: - result = newFloatNode(nkFloatLit, parseFloat(number)) - except ValueError: - raise newException(ExprParseError, &"Could not parse float value \"{number}\".") + getFloatNode(number, suffix) else: - case suffix - of "u", "U": - result = newNode(nkUintLit) - of "l", "L": - result = newNode(nkInt32Lit) - of "ul", "UL": - result = newNode(nkUint32Lit) - of "ll", "LL": - result = newNode(nkInt64Lit) - of "ull", "ULL": - result = newNode(nkUint64Lit) - else: - result = newNode(nkIntLit) + getIntNode(number, suffix) - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. - if number.contains(re"0[xX]"): - result.intVal = parseHexInt(number) - result.flags = {nfBase16} - elif number.contains(re"0[bB]"): - result.intVal = parseBinInt(number) - result.flags = {nfBase2} - elif number.contains(re"0[oO]"): - result.intVal = parseOctInt(number) - result.flags = {nfBase8} - else: - result.intVal = parseInt(number) - -proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val @@ -189,17 +180,17 @@ proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode = if result.kind != nkNone and prefix == "-": result = nkPrefix.newTree( - exprParser.state.getIdent("-"), + gState.getIdent("-"), result ) else: raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") -proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processCharacterLiteral(gState: State, node: TSNode): PNode = let val = node.val result = getCharLit(val[1 ..< val.len - 1]) -proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = +proc processStringLiteral(gState: State, node: TSNode): PNode = let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -215,9 +206,9 @@ proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode = result = newStrNode(nkStrLit, nimStr) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkInfix) let left = node[0] @@ -227,24 +218,24 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va case shiftSym of "<<": - result.add exprParser.state.getIdent("shl") + result.add gState.getIdent("shl") of ">>": - result.add exprParser.state.getIdent("shr") + result.add gState.getIdent("shr") else: raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - let leftNode = exprParser.processTSNode(left, typeofNode) + let leftNode = gState.processTSNode(left, typeofNode) # If the typeofNode is nil, set it # to be the leftNode because C's type coercion # happens left to right, and we want to emulate it if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -252,15 +243,15 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va rightNode ) -proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) for i in 0 ..< node.len(): - result.add(exprParser.processTSNode(node[i], typeofNode)) + result.add(gState.processTSNode(node[i], typeofNode)) -proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCast.newTree( - exprParser.processTSNode(node[0], typeofNode), - exprParser.processTSNode(node[1], typeofNode) + gState.processTSNode(node[0], typeofNode), + gState.processTSNode(node[1], typeofNode) ) proc getNimUnarySym(csymbol: string): string = @@ -292,7 +283,7 @@ proc getNimBinarySym(csymbol: string): string = else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") -proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) result = newNode(nkInfix) @@ -302,16 +293,16 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v binarySym = node.tsNodeChild(1).val.strip() nimSym = getNimBinarySym(binarySym) - result.add exprParser.state.getIdent(nimSym) - let leftNode = exprParser.processTSNode(left, typeofNode) + result.add gState.getIdent(nimSym) + let leftNode = gState.processTSNode(left, typeofNode) if typeofNode.isNil: typeofNode = nkCall.newTree( - exprParser.state.getIdent("typeof"), + gState.getIdent("typeof"), leftNode ) - let rightNode = exprParser.processTSNode(right, typeofNode) + let rightNode = gState.processTSNode(right, typeofNode) result.add leftNode result.add nkCall.newTree( @@ -319,7 +310,7 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v rightNode ) -proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = newNode(nkPar) let @@ -333,31 +324,31 @@ proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: va # Might be bad because we are overwriting the type # There's probably a better way of doing this if typeofNode.isNil: - typeofNode = exprParser.state.getIdent("int64") + typeofNode = gState.getIdent("int64") result.add nkPrefix.newTree( - exprParser.state.getIdent(unarySym), + gState.getIdent(unarySym), nkPar.newTree( nkCall.newTree( - exprParser.state.getIdent("int64"), - exprParser.processTSNode(child, typeofNode) + gState.getIdent("int64"), + gState.processTSNode(child, typeofNode) ) ) ) else: result.add nkPrefix.newTree( - exprParser.state.getIdent(nimSym), - exprParser.processTSNode(child, typeofNode) + gState.getIdent(nimSym), + gState.processTSNode(child, typeofNode) ) -proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len > 1: # Node has left and right children ie: (2 + 7) # Make sure the statement is of the same type as the left # hand argument, since some expressions return a differing # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(exprParser, node, typeofNode) + let binExpr = processBinaryExpression(gState, node, typeofNode) # Note that this temp var binExpr is needed for some reason, or else we get a segfault result = nkCall.newTree( typeofNode, @@ -366,37 +357,37 @@ proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeof elif node.len() == 1: # Node has only one child, ie -(20 + 7) - result = processUnaryExpression(exprParser, node, typeofNode) + result = processUnaryExpression(gState, node, typeofNode) else: raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") -proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = nkCall.newTree( - exprParser.state.getIdent("sizeof"), - exprParser.processTSNode(node[0], typeofNode) + gState.getIdent("sizeof"), + gState.processTSNode(node[0], typeofNode) ) -proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode = +proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = ## Handle all of the types of expressions here. This proc gets called recursively ## in the processX procs and will drill down to sub nodes. result = newNode(nkNone) let nodeName = node.getName() - techo "NODE: ", nodeName, ", VAL: ", node.val + decho "NODE: ", nodeName, ", VAL: ", node.val case nodeName of "number_literal": # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334 - result = exprParser.processNumberLiteral(node) + result = gState.processNumberLiteral(node) of "string_literal": # Input -> "foo\0\x42" # Output -> "foo\0" - result = exprParser.processStringLiteral(node) + result = gState.processStringLiteral(node) of "char_literal": # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences # Output -> - result = exprParser.processCharacterLiteral(node) + result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain # an ERROR node. If that's the case, they usually contain children with @@ -405,21 +396,21 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0) # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ if node.len == 1: - result = exprParser.processTSNode(node[0], typeofNode) + result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: result = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add exprParser.processTSNode(node[i], typeofNode) + result.add gState.processTSNode(node[i], typeofNode) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": # Input -> (IDENT - OTHERIDENT) # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int) - result = exprParser.processParenthesizedExpr(node, typeofNode) + result = gState.processParenthesizedExpr(node, typeofNode) of "sizeof_expression": # Input -> sizeof(char) # Output -> sizeof(cchar) - result = exprParser.processSizeofExpression(node, typeofNode) + result = gState.processSizeofExpression(node, typeofNode) # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", @@ -434,55 +425,61 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) - result = exprParser.processUnaryOrBinaryExpression(node, typeofNode) + result = gState.processUnaryOrBinaryExpression(node, typeofNode) of "shift_expression": # Input -> a >> b, a << b # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = exprParser.processShiftExpression(node, typeofNode) + result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) - result = exprParser.processCastExpression(node, typeofNode) + result = gState.processCastExpression(node, typeofNode) # Why are these node types named true/false? of "true", "false": # Input -> true, false # Output -> true, false - result = exprParser.state.parseString(node.val) + result = gState.parseString(node.val) of "type_descriptor", "sized_type_specifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc let ty = getType(node.val) if ty.len > 0: # If ty is not empty, one of C's builtin types has been found - result = exprParser.getIdent(ty, nskType, parent=node.getName()) + result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: - result = exprParser.getIdent(node.val, nskType, parent=node.getName()) + result = gState.getExprIdent(node.val, nskType, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) - result = exprParser.getIdent(node, parent=node.getName()) + result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - techo "NODE RESULT: ", result + decho "NODE RESULT: ", result -proc parseCExpression*(state: NimState, code: string, name = ""): PNode = +proc parseCExpression*(gState: State, code: string, name = ""): PNode = ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + result = newNode(nkNone) # This is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil - let exprParser = newExprParser(state, code, name) try: - withCodeAst(exprParser.code, exprParser.mode): - result = exprParser.processTSNode(root, tnode) + withCodeAst(gState.currentExpr, gState.mode): + result = gState.processTSNode(root, tnode) except ExprParseError as e: - techo e.msg + decho e.msg result = newNode(nkNone) except Exception as e: - techo "UNEXPECTED EXCEPTION: ", e.msg - result = newNode(nkNone) \ No newline at end of file + decho "UNEXPECTED EXCEPTION: ", e.msg + result = newNode(nkNone) + + # Clear the state + gState.currentExpr = "" + gState.currentTyCastName = "" \ No newline at end of file diff --git a/nimterop/globals.nim b/nimterop/globals.nim index b964da9..0d0b4cd 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,4 +1,4 @@ -import sequtils, sets, tables +import sequtils, sets, tables, strutils import regex @@ -93,6 +93,9 @@ type currentHeader*, impShort*, sourceFile*: string + # Used for the exprparser.nim module + currentExpr*, currentTyCastName*: string + data*: seq[tuple[name, val: string]] nodeBranch*: seq[string] @@ -119,6 +122,6 @@ when not declared(CIMPORT): else: gState.outputHandle.writeLine(args) - template decho*(str: untyped): untyped = + template decho*(args: varargs[string, `$`]): untyped = if gState.debug: - gecho str.getCommented() + gecho join(args, "").getCommented() \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index afb511d..98045bf 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -15,44 +15,16 @@ proc process(gState: State, path: string, astTable: AstTable) = else: gState.code = readFile(path) -<<<<<<< HEAD - doAssert gState.code.nBl, "Empty file or preprocessor error" - - if gState.mode == "c": - doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser" - elif gState.mode == "cpp": - doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" - else: - doAssert false, &"Invalid parser {gState.mode}" - - var - tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32) - root = tree.tsTreeRootNode() - - defer: - tree.tsTreeDelete() - - if gState.past: - gecho gState.printLisp(root) - elif gState.pnim: - if Feature.ast2 in gState.feature: - ast2.parseNim(gState, path, root) - else: - ast.parseNim(gState, path, root, astTable) - elif gState.preprocess: - gecho gState.code -======= withCodeAst(gState.code, gState.mode): if gState.past: gecho gState.printLisp(root) elif gState.pnim: if Feature.ast2 in gState.feature: - ast2.printNim(gState, path, root) + ast2.parseNim(gState, path, root) else: - ast.printNim(gState, path, root, astTable) + ast.parseNim(gState, path, root, astTable) elif gState.preprocess: gecho gState.code ->>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate # CLI processing with default values proc main( From c0977fa8271f462c20a29f4bb0976c638a534b45 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:15:40 -0600 Subject: [PATCH 16/24] Update code with comments and add debug expr proc --- nimterop/exprparser.nim | 143 +++++++++++++++++++++++++++++++++++++++- nimterop/getters.nim | 8 +-- 2 files changed, 144 insertions(+), 7 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 047a909..7acb375 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -35,6 +35,11 @@ type template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) +proc printDebugExpr*(gState: State, node: TSNode) = + if gState.debug: + gecho ("Input => " & node.val).getCommented() + gecho gState.currentExpr.printLisp(node).getCommented() + proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` ## @@ -187,10 +192,29 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") proc processCharacterLiteral(gState: State, node: TSNode): PNode = + # Input => 'G' + # + # (char_literal 1 1 3 "'G'") + # + # Output => 'G' + # + # nkCharLit("G") let val = node.val result = getCharLit(val[1 ..< val.len - 1]) proc processStringLiteral(gState: State, node: TSNode): PNode = + # Input => "\n\rfoobar\0\'" + # + # (string_literal 1 1 16 ""\n\rfoobar\0\'"" + # (escape_sequence 1 2 2 "\n") + # (escape_sequence 1 4 2 "\r") + # (escape_sequence 1 12 2 "\0") + # (escape_sequence 1 14 2 "\'") + # ) + # + # Output => "\n\cfoobar\x00\'" + # + # nkStrLit("\x0A\x0Dfoobar\x00\'") let nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] @@ -209,6 +233,26 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => a >> b + # + # (shift_expression 1 2 6 + # (identifier 1 2 1 "a") + # (identifier 1 7 1 "b") + # ) + # + # Output => a shr typeof(a)(b) + # + # nkInfix( + # nkIdent("shr"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let left = node[0] @@ -244,11 +288,56 @@ proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (a + b) + # + # (parenthesized_expression 1 1 7 + # (math_expression 1 2 5 + # (identifier 1 2 1 "a") + # (identifier 1 6 1 "b") + # ) + # ) + # + # Output => (typeof(a)(a + typeof(a)(b))) + # + # nkPar( + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkInfix( + # nkIdent("+"), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) + # ) + # ) result = newNode(nkPar) for i in 0 ..< node.len(): result.add(gState.processTSNode(node[i], typeofNode)) proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => (int)a + # + # (cast_expression 1 1 6 "(int)a" + # (type_descriptor 1 2 3 "int" + # (primitive_type 1 2 3 "int") + # ) + # (identifier 1 6 1 "a") + # ) + # + # Output => cast[cint](a) + # + # nkCast( + # nkIdent("cint"), + # nkIdent("a") + # ) result = nkCast.newTree( gState.processTSNode(node[0], typeofNode), gState.processTSNode(node[1], typeofNode) @@ -285,6 +374,27 @@ proc getNimBinarySym(csymbol: string): string = proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Node has left and right children ie: (2 + 7) + # + # Input => a == b + # + # (equality_expression 1 1 6 + # (identifier 1 1 1 "a") + # (identifier 1 6 1 "b") + # ) + # + # Output => a == typeof(a)(b) + # + # nkInfix( + # nkIdent("=="), + # nkIdent("a"), + # nkCall( + # nkCall( + # nkIdent("typeof"), + # nkIdent("a") + # ), + # nkIdent("b") + # ) + # ) result = newNode(nkInfix) let @@ -311,6 +421,20 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => !a + # + # (logical_expression 1 1 2 "!a" + # (identifier 1 2 1 "a") + # ) + # + # Output => (not a) + # + # nkPar( + # nkPrefix( + # nkIdent("not"), + # nkIdent("a") + # ) + # ) result = newNode(nkPar) let @@ -342,6 +466,7 @@ proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): ) proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) @@ -362,6 +487,20 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"") proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = + # Input => sizeof(int) + # + # (sizeof_expression 1 1 11 "sizeof(int)" + # (type_descriptor 1 8 3 "int" + # (primitive_type 1 8 3 "int") + # ) + # ) + # + # Output => sizeof(cint) + # + # nkCall( + # nkIdent("sizeof"), + # nkIdent("cint") + # ) result = nkCall.newTree( gState.getIdent("sizeof"), gState.processTSNode(node[0], typeofNode) @@ -385,8 +524,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Output -> "foo\0" result = gState.processStringLiteral(node) of "char_literal": - # Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences - # Output -> + # Input -> 'F', '\060' // Octal, '\x5A' // Hex, '\r' // escape sequences + # Output -> 'F', '0', 'Z', '\r' result = gState.processCharacterLiteral(node) of "expression_statement", "ERROR", "translation_unit": # Note that we're parsing partial expressions, so the TSNode might contain diff --git a/nimterop/getters.nim b/nimterop/getters.nim index b7744ce..121c8d5 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -436,13 +436,13 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n# " & offset & $pnode.kind & "(" case pnode.kind of nkCharLit: - result &= "'" & pnode.intVal.char & "')" + result &= ($pnode.intVal.char).escape & ")" of nkIntLit..nkUInt64Lit: result &= $pnode.intVal & ")" of nkFloatLit..nkFloat128Lit: result &= $pnode.floatVal & ")" of nkStrLit..nkTripleStrLit: - result &= "\"" & pnode.strVal & "\")" + result &= pnode.strVal.escape & ")" of nkSym: result &= $pnode.sym & ")" of nkIdent: @@ -460,16 +460,14 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string = result &= "\n" proc printDebug*(gState: State, node: TSNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug: gecho ("Input => " & gState.getNodeVal(node)).getCommented() gecho gState.printLisp(node).getCommented() proc printDebug*(gState: State, pnode: PNode) = - # This causes random segfaults for some reason on macOS Catalina if gState.debug and pnode.kind != nkNone: gecho ("Output => " & $pnode).getCommented() - gecho gState.printTree(pnode).getCommented() + gecho gState.printTree(pnode) # Compiler shortcuts From 2813dc61f22aecaaaa82054f144a48c4de520cf2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:45:19 -0600 Subject: [PATCH 17/24] Consolidate shift_expression into binary_expression and write more tests --- nimterop/exprparser.nim | 91 +++++++++-------------------------------- tests/include/tast2.h | 10 +++++ tests/tast2.nim | 14 ++++++- 3 files changed, 42 insertions(+), 73 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 7acb375..a59a546 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -232,61 +232,6 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode -proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = - # Input => a >> b - # - # (shift_expression 1 2 6 - # (identifier 1 2 1 "a") - # (identifier 1 7 1 "b") - # ) - # - # Output => a shr typeof(a)(b) - # - # nkInfix( - # nkIdent("shr"), - # nkIdent("a"), - # nkCall( - # nkCall( - # nkIdent("typeof"), - # nkIdent("a") - # ), - # nkIdent("b") - # ) - # ) - result = newNode(nkInfix) - let - left = node[0] - right = node[1] - - let shiftSym = node.tsNodeChild(1).val.strip() - - case shiftSym - of "<<": - result.add gState.getIdent("shl") - of ">>": - result.add gState.getIdent("shr") - else: - raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"") - - let leftNode = gState.processTSNode(left, typeofNode) - - # If the typeofNode is nil, set it - # to be the leftNode because C's type coercion - # happens left to right, and we want to emulate it - if typeofNode.isNil: - typeofNode = nkCall.newTree( - gState.getIdent("typeof"), - leftNode - ) - - let rightNode = gState.processTSNode(right, typeofNode) - - result.add leftNode - result.add nkCall.newTree( - typeofNode, - rightNode - ) - proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => (a + b) # @@ -369,6 +314,10 @@ proc getNimBinarySym(csymbol: string): string = result = csymbol of "%": result = "mod" + of "<<": + result = "shl" + of ">>": + result = "shr" else: raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"") @@ -419,6 +368,15 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) typeofNode, rightNode ) + if binarySym == "/": + # Special case. Nim's operators generally output + # the same type they take in, except for division. + # So we need to emulate C here and cast the whole + # expression to the type of the first arg + result = nkCall.newTree( + typeofNode, + result + ) proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode = # Input => !a @@ -469,17 +427,7 @@ proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions if node.len > 1: # Node has left and right children ie: (2 + 7) - - # Make sure the statement is of the same type as the left - # hand argument, since some expressions return a differing - # type than the input types (2/3 == float) - let binExpr = processBinaryExpression(gState, node, typeofNode) - # Note that this temp var binExpr is needed for some reason, or else we get a segfault - result = nkCall.newTree( - typeofNode, - binexpr - ) - + result = processBinaryExpression(gState, node, typeofNode) elif node.len() == 1: # Node has only one child, ie -(20 + 7) result = processUnaryExpression(gState, node, typeofNode) @@ -553,8 +501,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # binary_expression from the new treesitter upgrade should work here # once we upgrade of "math_expression", "logical_expression", "relational_expression", - "bitwise_expression", "equality_expression", "binary_expression": - # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b + "bitwise_expression", "equality_expression", "binary_expression", + "shift_expression": + # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b # Output -> # typeof(a)(a == typeof(a)(b)) # typeof(a)(a != typeof(a)(b)) @@ -564,11 +513,9 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # typeof(a)(a > typeof(a)(b)) # typeof(a)(a <= typeof(a)(b)) # typeof(a)(a >= typeof(a)(b)) + # a shr typeof(a)(b) + # a shl typeof(a)(b) result = gState.processUnaryOrBinaryExpression(node, typeofNode) - of "shift_expression": - # Input -> a >> b, a << b - # Output -> a shr typeof(a)(b), a shl typeof(a)(b) - result = gState.processShiftExpression(node, typeofNode) of "cast_expression": # Input -> (int) a # Output -> cast[cint](a) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 53b4d64..e72a3b8 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,6 +24,16 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 +#define a 100 +#define b 200 +#define EQ1 a <= b +#define EQ2 a >= b +#define EQ3 a > b +#define EQ4 a < b +#define EQ5 a != b +#define EQ6 a == b + +#define SIZEOF sizeof(char) #define NULLCHAR '\0' #define OCTCHAR '\012' diff --git a/tests/tast2.nim b/tests/tast2.nim index e82430b..57bfc90 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,6 +115,18 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) +assert a == 100 +assert b == 200 + +assert EQ1 == (a <= b) +assert EQ2 == (a >= b) +assert EQ3 == (a > b) +assert EQ4 == (a < b) +assert EQ5 == (a != b) +assert EQ6 == (a == b) + +assert SIZEOF == 1 + assert COERCE == 645635670332'u64 assert COERCE2 == 645635670332'i64 @@ -303,7 +315,7 @@ var a21p: A21p a21p = addr a20 assert A22 is object -testFields(A22, "f1|f2!ptr ptr cint|array[type(123)(255), ptr cint]") +testFields(A22, "f1|f2!ptr ptr cint|array[123 + type(123)(132), ptr cint]") checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] From 9e799ee3b19ac152d1c2403c6af1e3806e1a31fe Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 16:53:52 -0600 Subject: [PATCH 18/24] Fix tmath tests Modify tmath.nim to have one skip symbol Fix macro expansion with common value Fix tmath again --- tests/include/tast2.h | 16 ++++++++-------- tests/tast2.nim | 16 ++++++++-------- tests/tmath.nim | 6 ++++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index e72a3b8..815791f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -24,14 +24,14 @@ extern "C" { #define MATHEXPR (1 + 2/3*20 - 100) #define ANDEXPR (100 & 11000) #define CASTEXPR (char) 34 -#define a 100 -#define b 200 -#define EQ1 a <= b -#define EQ2 a >= b -#define EQ3 a > b -#define EQ4 a < b -#define EQ5 a != b -#define EQ6 a == b +#define AVAL 100 +#define BVAL 200 +#define EQ1 AVAL <= BVAL +#define EQ2 AVAL >= BVAL +#define EQ3 AVAL > BVAL +#define EQ4 AVAL < BVAL +#define EQ5 AVAL != BVAL +#define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) diff --git a/tests/tast2.nim b/tests/tast2.nim index 57bfc90..da7fafa 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -115,15 +115,15 @@ assert ULLEXPR == (1234.uint64 shl 3) assert LEXPR == (1234.int32 shl 4) assert LLEXPR == (1234.int64 shl 5) -assert a == 100 -assert b == 200 +assert AVAL == 100 +assert BVAL == 200 -assert EQ1 == (a <= b) -assert EQ2 == (a >= b) -assert EQ3 == (a > b) -assert EQ4 == (a < b) -assert EQ5 == (a != b) -assert EQ6 == (a == b) +assert EQ1 == (AVAL <= BVAL) +assert EQ2 == (AVAL >= BVAL) +assert EQ3 == (AVAL > BVAL) +assert EQ4 == (AVAL < BVAL) +assert EQ5 == (AVAL != BVAL) +assert EQ6 == (AVAL == BVAL) assert SIZEOF == 1 diff --git a/tests/tmath.nim b/tests/tmath.nim index bbf7abd..b8477c1 100644 --- a/tests/tmath.nim +++ b/tests/tmath.nim @@ -14,9 +14,11 @@ when defined(windows): static: when (NimMajor, NimMinor, NimPatch) < (1, 0, 0): - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE", "FP_ILOGB0", "FP_ILOGBNAN"] + # FP_ILOGB0 and FP_ILOGBNAN are casts that are unsupported + # on lower Nim VMs + cSkipSymbol @["math_errhandling", "FP_ILOGB0", "FP_ILOGBNAN"] else: - cSkipSymbol @["mingw_choose_expr", "EXCEPTION_DEFINED", "COMPLEX_DEFINED", "matherr", "HUGE"] + cSkipSymbol @["math_errhandling"] cDebug() cDisableCaching() cAddStdDir() From 073dc5d35a70270c49234842bfbb7f3e608c539e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 19:21:57 -0600 Subject: [PATCH 19/24] Add hack for skipping types as root nodes --- nimterop/ast2.nim | 30 ++++++++++++++++++++++++++---- nimterop/exprparser.nim | 21 +++++++++++++-------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4fec237..bb41ebe 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,9 +4,9 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/api +import "."/treesitter/[api, c, cpp] -import "."/[globals, getters, exprparser, comphelp] +import "."/[globals, getters, exprparser, comphelp, tshelp] proc getPtrType*(str: string): string = result = case str: @@ -99,8 +99,30 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) - valident = - gState.parseCExpression(val) + var valident = newNode(nkNone) + + withCodeAst(val, gState.mode): + # This section is a hack for determining that the first + # node is a type, which shouldn't be accepted by a const + # def section. Need to replace this with some other mechanism + # to handle type aliases + var maybeTyNode: TSNode + # Take the very first node, which may be 2 levels + # down if there is an error node + if root.len > 0 and root[0].getName() == "ERROR": + maybeTyNode = root[0][0] + elif root.len > 0: + maybeTyNode = root[0] + + if not maybeTyNode.isNil: + let name = maybeTyNode.getName() + case name + of "type_descriptor", "sized_type_specifier": + discard + else: + # Can't do gState.parseCExpression(root) here for some reason? + # get a SEGFAULT if we use root + valident = gState.parseCExpression(val) if name.Bl: # Name skipped or overridden since blank diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index a59a546..6e102ea 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -547,18 +547,15 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = decho "NODE RESULT: ", result -proc parseCExpression*(gState: State, code: string, name = ""): PNode = - ## Convert the C string to a nim PNode tree - gState.currentExpr = code - gState.currentTyCastName = name +proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode = + ## Parse a c expression from a root ts node - result = newNode(nkNone) - # This is used for keeping track of the type of the first + # This var is used for keeping track of the type of the first # symbol used for type casting var tnode: PNode = nil + result = newNode(nkNone) try: - withCodeAst(gState.currentExpr, gState.mode): - result = gState.processTSNode(root, tnode) + result = gState.processTSNode(codeRoot, tnode) except ExprParseError as e: decho e.msg result = newNode(nkNone) @@ -566,6 +563,14 @@ proc parseCExpression*(gState: State, code: string, name = ""): PNode = decho "UNEXPECTED EXCEPTION: ", e.msg result = newNode(nkNone) +proc parseCExpression*(gState: State, code: string, name = ""): PNode = + ## Convert the C string to a nim PNode tree + gState.currentExpr = code + gState.currentTyCastName = name + + withCodeAst(gState.currentExpr, gState.mode): + result = gState.parseCExpression(root, name) + # Clear the state gState.currentExpr = "" gState.currentTyCastName = "" \ No newline at end of file From cc460b2779fc8b2035c19e0ba8f460cf124d474f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:20:05 -0600 Subject: [PATCH 20/24] Add skippedSymbols for determining if a type has been skipped --- nimterop/ast2.nim | 3 +++ nimterop/exprparser.nim | 25 +++++++++++++------------ nimterop/globals.nim | 3 +++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index bb41ebe..2ae4a70 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -41,6 +41,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS result = pnode[0][0] else: gecho &"\n# $1'{origname}' skipped" % skind + gState.skippedSyms.incl origname if gState.debug: gState.skipStr &= &"\n{gState.getNodeVal(node)}" @@ -99,6 +100,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = fval else: gState.getNodeVal(node[1]) + var valident = newNode(nkNone) withCodeAst(val, gState.mode): @@ -151,6 +153,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = gecho &"# const '{origname}' is duplicate, skipped" else: gecho &"# const '{origname}' has invalid value '{val}'" + gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = # Add a const to the AST diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 6e102ea..1f6decc 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -45,16 +45,17 @@ proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = " ## ## Returns PNode(nkNone) if the identifier is blank result = newNode(nkNone) - var ident = identName - if ident != "_": - # Process the identifier through cPlugin - ident = gState.getIdentifier(ident, kind, parent) - if kind == nskType: - result = gState.getIdent(ident) - elif ident.nBl and ident in gState.constIdentifiers: - if gState.currentTyCastName.nBl: - ident = ident & "." & gState.currentTyCastName - result = gState.getIdent(ident) + if identName notin gState.skippedSyms: + var ident = identName + if ident != "_": + # Process the identifier through cPlugin + ident = gState.getIdentifier(ident, kind, parent) + if kind == nskType: + result = gState.getIdent(ident) + elif ident.nBl and ident in gState.constIdentifiers: + if gState.currentTyCastName.nBl: + ident = ident & "." & gState.currentTyCastName + result = gState.getIdent(ident) proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode = ## Gets a cPlugin transformed identifier from `identName` @@ -534,8 +535,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(ty, nskType, parent=node.getName()) else: result = gState.getExprIdent(node.val, nskType, parent=node.getName()) - if result.kind == nkNone: - raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") + if result.kind == nkNone: + raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"") of "identifier": # Input -> IDENT # Output -> IDENT (if found in sym table, else error) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 0d0b4cd..d433ab5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,6 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] + # All symbols that have been skipped + skippedSyms*: HashSet[string] + # Legacy ast fields, remove when ast2 becomes default constStr*, enumStr*, procStr*, typeStr*: string From 3e28501826f5d655ad3797cbf29009082be126e1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 20:59:21 -0600 Subject: [PATCH 21/24] Fix comments breaking code --- nimterop/exprparser.nim | 14 ++++++++++++-- tests/include/tast2.h | 4 ++-- tests/tpcre.nim | 1 - 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 1f6decc..e9ee502 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,9 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - result = newNode(nkStmtListExpr) + let res = newNode(nkStmtListExpr) for i in 0 ..< node.len: - result.add gState.processTSNode(node[i], typeofNode) + let node = gState.processTSNode(node[i], typeofNode) + if node.kind != nkNone: + res.add node + if res.len == 1: + result = res[0] + elif res.len > 1: + result = res + else: + result = newNode(nkNone) else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": @@ -543,6 +551,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = result = gState.getExprIdent(node, parent=node.getName()) if result.kind == nkNone: raise newException(ExprParseError, &"Missing identifier \"{node.val}\"") + of "comment": + discard else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 815791f..42c852f 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,8 +35,8 @@ extern "C" { #define SIZEOF sizeof(char) -#define NULLCHAR '\0' -#define OCTCHAR '\012' +#define NULLCHAR '\0'/* comments should not break things*/ +#define OCTCHAR '\012' // nor should this comment #define HEXCHAR '\xFE' #define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 3bbd1ba..c8e8059 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -7,7 +7,6 @@ const pcreH = baseDir/"pcre.h.in" static: - cSkipSymbol @["PCRE_UCHAR16", "PCRE_UCHAR32"] if not pcreH.fileExists(): downloadUrl("https://github.com/svn2github/pcre/raw/master/pcre.h.in", baseDir) cDebug() From 4e687dd80783e8ff7bca9ba8ba6ba61d3a933fc2 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 21:52:34 -0600 Subject: [PATCH 22/24] Don't support multiple base nodes yet --- nimterop/exprparser.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index e9ee502..c74f0b6 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -486,17 +486,17 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = if node.len == 1: result = gState.processTSNode(node[0], typeofNode) elif node.len > 1: - let res = newNode(nkStmtListExpr) + var nodes: seq[PNode] for i in 0 ..< node.len: - let node = gState.processTSNode(node[i], typeofNode) - if node.kind != nkNone: - res.add node - if res.len == 1: - result = res[0] - elif res.len > 1: - result = res - else: - result = newNode(nkNone) + let subNode = gState.processTSNode(node[i], typeofNode) + if subNode.kind != nkNone: + nodes.add(subNode) + # Multiple nodes can get tricky. Don't support them yet, unless they + # have at most one valid node + if nodes.len > 1: + raise newException(ExprParseError, &"Node type \"{nodeName}\" with val ({node.val}) has more than one non empty node") + if nodes.len == 1: + result = nodes[0] else: raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children") of "parenthesized_expression": From 322a0031984cefc278d2a377daa27bffdcbcae73 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 23 Apr 2020 22:11:39 -0600 Subject: [PATCH 23/24] Add test for not supported nodes --- tests/include/tast2.h | 2 ++ tests/tast2.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 42c852f..b47a801 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -34,6 +34,8 @@ extern "C" { #define EQ6 AVAL == BVAL #define SIZEOF sizeof(char) +#define REG_STR "regular string" +#define NOTSUPPORTEDSTR "not a " REG_STR #define NULLCHAR '\0'/* comments should not break things*/ #define OCTCHAR '\012' // nor should this comment diff --git a/tests/tast2.nim b/tests/tast2.nim index da7fafa..4cfbeac 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -109,6 +109,8 @@ assert C == 0x10 assert D == "hello" assert E == 'c' +assert not defined(NOTSUPPORTEDSTR) + assert UEXPR == (1234.uint shl 1) assert ULEXPR == (1234.uint32 shl 2) assert ULLEXPR == (1234.uint64 shl 3) From 89c10c4b25226a88062d01e6bf57a9e9146920c5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 25 Apr 2020 14:25:42 -0600 Subject: [PATCH 24/24] Address some PR comments --- nimterop/ast2.nim | 12 +++++++----- nimterop/globals.nim | 4 +++- nimterop/tshelp.nim | 14 ++++++-------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 2ae4a70..474ec69 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,7 +4,7 @@ import options as opts import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/[api, c, cpp] +import "."/treesitter/api import "."/[globals, getters, exprparser, comphelp, tshelp] @@ -1391,7 +1391,9 @@ proc addEnum(gState: State, node: TSNode) = # Create const for fields var fnames: HashSet[string] - fvalSections: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + # Hold all of field information so that we can add all of them + # after the const identifiers has been updated + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1410,9 +1412,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fvalSections.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]))) else: - fvalSections.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode))) fnames.incl fname prev = fname @@ -1422,7 +1424,7 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fvalSections: + for (fname, fval, cexprNode) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d433ab5..5db17a3 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -76,7 +76,9 @@ type # All const names for enum casting constIdentifiers*: HashSet[string] - # All symbols that have been skipped + # All symbols that have been skipped due to + # being unwrappable or the user provided + # override is blank skippedSyms*: HashSet[string] # Legacy ast fields, remove when ast2 becomes default diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index f234bc0..109321c 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,11 +1,9 @@ -template withCodeAst*(inputCode: string, inputMode: string, body: untyped): untyped = - ## A simple template to inject the TSNode into a body of code +import "."/treesitter/[c, cpp] - # This section is needed to be able to reference - # mode in strformat calls - let - code = inputCode - mode {.inject.} = inputMode +template withCodeAst*(code: string, mode: string, body: untyped): untyped = + ## A simple template to inject the TSNode into a body of code + mixin treeSitterC + mixin treeSitterCpp var parser = tsParserNew() defer: @@ -18,7 +16,7 @@ template withCodeAst*(inputCode: string, inputMode: string, body: untyped): unty elif mode == "cpp": doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser" else: - doAssert false, &"Invalid parser {mode}" + doAssert false, "Invalid parser " & mode var tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32)