From cfd18e10083ad00274b2e16afbcf0a292bc78549 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 29 Dec 2019 04:46:19 -0600 Subject: [PATCH 01/22] New Nim AST based backend - #defines working --- nimterop/ast2.nim | 133 ++++++++++++++++++++ nimterop/getters.nim | 292 ++++++++++++++++++++++--------------------- nimterop/globals.nim | 29 ++--- nimterop/nim.cfg | 1 + nimterop/toast.nim | 20 +-- 5 files changed, 298 insertions(+), 177 deletions(-) create mode 100644 nimterop/ast2.nim create mode 100644 nimterop/nim.cfg diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim new file mode 100644 index 0000000..dc62d4f --- /dev/null +++ b/nimterop/ast2.nim @@ -0,0 +1,133 @@ +import macros, os, strutils, tables, times + +import compiler/[ast, idents, options, renderer] + +import "."/treesitter/api + +import "."/[compat, globals, getters] + +proc addConst(nimState: NimState, node: TSNode) = + echo "Const: " & nimState.getNodeVal(node) + if node.len() >= 2: + let + constDef = newNode(nkConstDef) + (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) + ident = nimState.getIdent(name, info) + val = nimState.getNodeVal(node[1]).getLit() + if val.kind != nkNilLit: + constDef.add ident + constDef.add newNode(nkEmpty) + constDef.add val + + nimState.constSection.add constDef + +proc addType(nimState: NimState, node: TSNode) = + echo "Type: " & nimState.getNodeVal(node) + +proc addEnum(nimState: NimState, node: TSNode) = + echo "Enum: " & nimState.getNodeVal(node) + +proc addProc(nimState: NimState, node: TSNode) = + echo "Proc: " & nimState.getNodeVal(node) + +proc processNode(nimState: NimState, node: TSNode): bool = + result = true + + case node.getName() + of "preproc_def": + nimState.addConst(node) + of "type_definition": + if node.inTree("struct_specifier"): + nimState.addType(node) + elif node.inTree("enum_specifier"): + nimState.addEnum(node) + else: + # Unknown type + result = false + of "struct_specifier": + nimState.addType(node) + of "enum_specifier": + nimState.addEnum(node) + of "declaration": + nimState.addProc(node) + else: + # Unknown + result = false + +proc searchTree(nimState: NimState, root: TSNode) = + # Search AST generated by tree-sitter for recognized elements + var + node = root + nextnode: TSNode + depth = 0 + processed = false + + while true: + if not node.tsNodeIsNull() and depth > -1: + processed = nimState.processNode(node) + else: + break + + if not processed and node.len() != 0: + nextnode = node[0] + depth += 1 + else: + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.tsNodeIsNull(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if node == root: + break + if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc printNimHeader*() = + echo """# Generated at $1 +# Command line: +# $2 $3 + +{.hint[ConvFromXtoItselfNotNeeded]: off.} + +import nimterop/types +""" % [$now(), getAppFilename(), commandLineParams().join(" ")] + +proc printNim*(gState: State, fullpath: string, root: TSNode) = + var + nimState = new(NimState) + fp = fullpath.replace("\\", "/") + + nimState.identifiers = newTable[string, string]() + + nimState.gState = gState + nimState.currentHeader = getCurrentHeader(fullpath) + nimState.sourceFile = fullpath + + # Nim compiler objects + nimState.identCache = newIdentCache() + nimState.config = newConfigRef() + + nimState.constSection = newNode(nkConstSection) + nimState.enumSection = newNode(nkStmtList) + nimState.procSection = newNode(nkStmtList) + nimState.typeSection = newNode(nkTypeSection) + + nimState.searchTree(root) + + var + tree = newNode(nkStmtList) + tree.add nimState.enumSection + tree.add nimState.constSection + tree.add nimState.typeSection + tree.add nimState.procSection + + echo tree.renderTree() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index ef647e6..fbbead2 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,6 +2,8 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex +import compiler/[ast, idents, lineinfos, msgs, pathutils] + import "."/[build, compat, globals, plugin, treesitter/api] const gReserved = """ @@ -26,6 +28,8 @@ when while xor yield""".split(Whitespace).toSet() +# Types related + const gTypeMap* = { # char "char": "cchar", @@ -91,6 +95,21 @@ proc getType*(str: string): string = if gTypeMap.hasKey(result): result = gTypeMap[result] +proc getPtrType*(str: string): string = + result = case str: + of "ptr cchar": + "cstring" + of "ptr ptr cchar": + "ptr cstring" + of "ptr object": + "pointer" + of "ptr ptr object": + "ptr pointer" + else: + str + +# Identifier related + proc checkIdentifier(name, kind, parent, origName: string) = let parentStr = if parent.nBl: parent & ":" else: "" @@ -165,6 +184,8 @@ proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool nimState.identifiers[nimName] = name result = true +# Overrides related + proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = doAssert name.nBl, "Blank identifier error" @@ -189,34 +210,96 @@ proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = for i in nimState.gState.onSymbolOverrideFinal(typ): result &= "\n" & nimState.getOverride(i, kind) -proc getPtrType*(str: string): string = - result = case str: - of "ptr cchar": - "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": - "pointer" - of "ptr ptr object": - "ptr pointer" - else: - str - -proc getLit*(str: string): string = +proc getLit*(str: string): PNode = # Used to convert #define literals into const let str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal - str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal - str.contains(re"^'[[:ascii:]]'$") or # char - str.contains(re"""^"[[:ascii:]]+"$"""): # char * - return str + if str.contains(re"^[\-]?[\d]+$"): # decimal + result = newIntNode(nkIntLit, parseInt(str)) + + elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float + result = newFloatNode(nkFloatLit, parseFloat(str)) + + elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + result = newIntNode(nkIntLit, parseHexInt(str)) + + elif str.contains(re"^'[[:ascii:]]'$"): # char + result = newNode(nkCharLit) + result.intVal = str[1].int64 + + elif str.contains(re"""^"[[:ascii:]]+"$"""): # char * + result = newStrNode(nkStrLit, str[1 .. ^2]) + + else: + result = newNode(nkNilLit) + +# TSNode shortcuts + +proc len*(node: TSNode): uint = + if not node.tsNodeIsNull: + result = node.tsNodeNamedChildCount().uint + +proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = + if i < node.len(): + result = node.tsNodeNamedChild(i.uint32) + +proc getName*(node: TSNode): string {.inline.} = + if not node.tsNodeIsNull: + return $node.tsNodeType() proc getNodeVal*(nimState: NimState, node: TSNode): string = - return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + if not node.tsNodeIsNull: + return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + +proc getAtom*(node: TSNode): TSNode = + if not node.tsNodeIsNull: + # Get child node which is topmost atom + if node.getName() in gAtoms: + return node + elif node.len() != 0: + return node[0].getAtom() + +proc getPtrCount*(node: TSNode): string = + if not node.tsNodeIsNull: + # Get number of ptr nodes in tree + var + cnode = node + while "pointer_declarator" in cnode.getName(): + result &= "ptr " + if cnode.len() != 0: + cnode = cnode[0] + else: + break + +proc getDeclarator*(node: TSNode): TSNode = + if not node.tsNodeIsNull: + # Return if child is a function or array declarator + if node.getName() in ["function_declarator", "array_declarator"]: + return node + elif node.len() != 0: + return node[0].getDeclarator() + +proc inTree*(node: TSNode, ntype: string): bool = + # Search for node type in tree - first children + result = false + var + cnode = node + while not cnode.tsNodeIsNull: + if cnode.getName() == ntype: + return true + cnode = cnode[0] + +proc inChildren*(node: TSNode, ntype: string): bool = + # Search for node type in immediate children + result = false + for i in 0 ..< node.len(): + if (node[i]).getName() == ntype: + result = true + break proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node result.line = 1 result.col = 1 for i in 0 .. node.tsNodeStartByte().int-1: @@ -225,6 +308,54 @@ proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = result.line += 1 result.col += 1 +proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = + for i in 0 ..< node.len(): + if node.getName() != "comment": + result += 1 + +proc getPxName*(node: TSNode, offset: int): string = + # Get the xth (grand)parent of the node + var + np = node + count = 0 + + while not np.tsNodeIsNull() and count < offset: + np = np.tsNodeParent() + count += 1 + + if count == offset and not np.tsNodeIsNull(): + return np.getName() + +# Compiler shortcuts + +proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = + # Get Nim equivalent line:col info from node + let + (line, col) = nimState.gState.getLineCol(node) + + result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col) + +proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode = + # Get ident PNode for name + info + let + exp = getIdent(nimState.identCache, "*") + ident = getIdent(nimState.identCache, name) + + if exported: + result = newNode(nkPostfix) + result.add newIdentNode(exp, info) + result.add newIdentNode(ident, info) + else: + result = newIdentNode(ident, info) + +proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): + tuple[name: string, info: TLineInfo] = + # Shortcut to get identifier name and info (node value and line:col) + let + name = nimState.getNodeVal(node) + result.name = nimState.getIdentifier(name, kind, parent) + result.info = nimState.getLineInfo(node) + proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) @@ -291,91 +422,6 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string = replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1"). removeStatic() -converter toString*(kind: Kind): string = - return case kind: - of exactlyOne: - "" - of oneOrMore: - "+" - of zeroOrMore: - "*" - of zeroOrOne: - "?" - of orWithNext: - "!" - -converter toKind*(kind: string): Kind = - return case kind: - of "+": - oneOrMore - of "*": - zeroOrMore - of "?": - zeroOrOne - of "!": - orWithNext - else: - exactlyOne - -proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool] = - if name[0] == '^': - result.recursive = true - result.name = name[1 .. ^1] - else: - result.name = name - result.kind = $name[^1] - - if result.kind != exactlyOne: - result.name = result.name[0 .. ^2] - -proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - if $node.tsNodeType() != "comment": - result += 1 - -proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - let - name = $node.tsNodeNamedChild(i).tsNodeType() - - if name != "comment": - result.add(name) - -proc getRegexForAstChildren*(ast: ref Ast): string = - result = "^" - for i in 0 .. ast.children.len-1: - let - kind: string = ast.children[i].kind - begin = if result[^1] == '|': "" else: "(?:" - case kind: - of "!": - result &= &"{begin}{ast.children[i].name}|" - else: - result &= &"{begin}{ast.children[i].name}){kind}" - result &= "$" - -proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = - for i in 0 .. ast.children.len-1: - if name in ast.children[i].name.split("|"): - return ast.children[i] - - if ast.children.len == 1 and ast.children[0].name == ".": - return ast.children[0] - -proc getPxName*(node: TSNode, offset: int): string = - var - np = node - count = 0 - - while not np.tsNodeIsNull() and count < offset: - np = np.tsNodeParent() - count += 1 - - if count == offset and not np.tsNodeIsNull(): - return $np.tsNodeType() - proc getNimExpression*(nimState: NimState, expr: string): string = var clean = expr.multiReplace([("\n", " "), ("\r", "")]) @@ -435,42 +481,6 @@ proc getSplitComma*(joined: seq[string]): seq[string] = for i in joined: result = result.concat(i.split(",")) -proc getHeader*(nimState: NimState): string = - result = - if nimState.gState.dynlib.Bl: - &", header: {nimState.currentHeader}" - else: - "" - -proc getDynlib*(nimState: NimState): string = - result = - if nimState.gState.dynlib.nBl: - &", dynlib: {nimState.gState.dynlib}" - else: - "" - -proc getImportC*(nimState: NimState, origName, nimName: string): string = - if nimName != origName: - result = &"importc: \"{origName}\"{nimState.getHeader()}" - else: - result = nimState.impShort - -proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = - result = "" - for pragma in pragmas.items(): - if pragma.nBl: - result &= pragma & ", " - if result.nBl: - result = " {." & result[0 .. ^3] & ".}" - - result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C") - - let - dy = nimState.getDynlib() - - if ", cdecl" in result and dy.nBl: - result = result.replace(".}", dy & ".}") - proc getComments*(nimState: NimState, strip = false): string = if not nimState.gState.nocomments and nimState.commentStr.nBl: result = "\n" & nimState.commentStr diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 7c512b1..d29ad6c 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,6 +1,6 @@ import sequtils, sets, tables -import regex +import compiler/[ast, idents, options] import "."/plugin @@ -34,24 +34,6 @@ const ].concat(toSeq(gExpressions.items)) type - Kind = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - when not declared(CIMPORT): - tonim*: proc (ast: ref Ast, node: TSNode, nimState: NimState) - regex*: Regex - - AstTable {.used.} = TableRef[string, seq[ref Ast]] - State = ref object compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string] @@ -65,7 +47,12 @@ type NimState {.used.} = ref object identifiers*: TableRef[string, string] - commentStr*, constStr*, debugStr*, enumStr*, procStr*, skipStr*, typeStr*: string + commentStr*, debugStr*, skipStr*: string + + # Nim compiler objects + constSection*, enumSection*, procSection*, typeSection*: PNode + identCache*: IdentCache + config*: ConfigRef gState*: State @@ -92,5 +79,5 @@ type CompileMode = enum const modeDefault {.used.} = $cpp # TODO: USE this everywhere relevant when not declared(CIMPORT): - export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, + export gAtoms, gExpressions, gEnumVals, State, NimState, nBl, Bl, CompileMode, modeDefault diff --git a/nimterop/nim.cfg b/nimterop/nim.cfg new file mode 100644 index 0000000..913df8c --- /dev/null +++ b/nimterop/nim.cfg @@ -0,0 +1 @@ +--path:"$nim" diff --git a/nimterop/toast.nim b/nimterop/toast.nim index c26b460..8c5aaa2 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,7 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast, compat, globals, getters, grammar] +import "."/[ast2, compat, globals, getters] proc printLisp(gState: State, root: TSNode) = var @@ -53,7 +53,7 @@ proc printLisp(gState: State, root: TSNode) = if node == root: break -proc process(gState: State, path: string, astTable: AstTable) = +proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" var @@ -93,7 +93,7 @@ proc process(gState: State, path: string, astTable: AstTable) = if gState.past: gState.printLisp(root) elif gState.pnim: - gState.printNim(path, root, astTable) + gState.printNim(path, root) elif gState.preprocess: echo gState.code @@ -109,7 +109,6 @@ proc main( nocomments = false, output = "", past = false, - pgrammar = false, pluginSourcePath: string = "", pnim = false, prefix: seq[string] = @[], @@ -176,19 +175,12 @@ proc main( doAssert outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}" stdout = outputHandle - # Process grammar into AST - let - astTable = parseGrammar() - - if pgrammar: - # Print AST of grammar - astTable.printGrammar() - elif source.nBl: + if source.nBl: # Print source after preprocess or Nim output if gState.pnim: printNimHeader() for src in source: - gState.process(src.expandSymlinkAbs(), astTable) + gState.process(src.expandSymlinkAbs()) when not defined(windows): # Restore stdout @@ -257,7 +249,6 @@ when isMainModule: "nocomments": "exclude top-level comments from output", "output": "file to output content - default stdout", "past": "print AST output", - "pgrammar": "print grammar", "pluginSourcePath": "Nim file to build and load as a plugin", "pnim": "print Nim output", "preprocess": "run preprocessor on header", @@ -276,7 +267,6 @@ when isMainModule: "nocomments": 'c', "output": 'o', "past": 'a', - "pgrammar": 'g', "pnim": 'n', "prefix": 'E', "preprocess": 'p', From 127aa4688e4a98a5caa51b4567ea02b27398ed60 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 1 Jan 2020 16:37:33 -0600 Subject: [PATCH 02/22] Print debug, more comments --- config.nims | 5 ++- nimterop/ast2.nim | 75 ++++++++++++++++++++++++++++++++--- nimterop/getters.nim | 93 +++++++++++++++++++++++++++++++++++++++++++- nimterop/nim.cfg | 1 - nimterop/toast.nim | 51 +----------------------- 5 files changed, 166 insertions(+), 59 deletions(-) delete mode 100644 nimterop/nim.cfg diff --git a/config.nims b/config.nims index 54a9a06..d312c07 100644 --- a/config.nims +++ b/config.nims @@ -9,4 +9,7 @@ when defined(Windows): switch("gc", "markAndSweep") # Retain stackTrace for clear errors -switch("stackTrace", "on") \ No newline at end of file +switch("stackTrace", "on") + +# Path to compiler +switch("path", "$nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dc62d4f..8790e6f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,34 +1,97 @@ import macros, os, strutils, tables, times -import compiler/[ast, idents, options, renderer] +import compiler/[ast, astalgo, idents, options, renderer] import "."/treesitter/api import "."/[compat, globals, getters] proc addConst(nimState: NimState, node: TSNode) = - echo "Const: " & nimState.getNodeVal(node) - if node.len() >= 2: + # (preproc_def + # (identifier) + # (preproc_arg) + # ) + nimState.printDebug(node) + + if node[0].getName() == "identifier" and + node[1].getName() == "preproc_arg": let constDef = newNode(nkConstDef) + + # node[0] = identifier = const name (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) + # TODO - check blank and override ident = nimState.getIdent(name, info) + + # node[1] = preproc_arg = value val = nimState.getNodeVal(node[1]).getLit() + + # If supported literal if val.kind != nkNilLit: + # nnkConstDef( + # nnkPostfix( + # nnkIdent("*"), + # nnkIdent(name) + # ), + # nnkEmpty(), + # nnkXLit(val) + # ) constDef.add ident constDef.add newNode(nkEmpty) constDef.add val + # nnkConstSection.add nimState.constSection.add constDef + nimState.printDebug(constDef) + proc addType(nimState: NimState, node: TSNode) = - echo "Type: " & nimState.getNodeVal(node) + # CASE1: + # + # typedef struct X Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_identifier) + # ) + # + # CASE 2 + # + # typedef struct X *Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator + # (type_identifier) + # ) + # ) + # + nimState.printDebug(node) + + let + typeDef = newNode(nkTypeDef) + + fdlist = node.inTree("field_declaration_list") + + identNode = if fdlist: node[1] else: node + + (name, info) = nimState.getNameInfo(identNode.getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info) + + typeNode = + if fdlist: node[0] else: node[1] + proc addEnum(nimState: NimState, node: TSNode) = - echo "Enum: " & nimState.getNodeVal(node) + nimState.printDebug(node) proc addProc(nimState: NimState, node: TSNode) = - echo "Proc: " & nimState.getNodeVal(node) + nimState.printDebug(node) proc processNode(nimState: NimState, node: TSNode): bool = result = true diff --git a/nimterop/getters.nim b/nimterop/getters.nim index fbbead2..8e03282 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, lineinfos, msgs, pathutils] +import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] import "."/[build, compat, globals, plugin, treesitter/api] @@ -221,6 +221,7 @@ proc getLit*(str: string): PNode = elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float result = newFloatNode(nkFloatLit, parseFloat(str)) + # TODO - hex becomes int on render elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal result = newIntNode(nkIntLit, parseHexInt(str)) @@ -326,6 +327,96 @@ proc getPxName*(node: TSNode, offset: int): string = if count == offset and not np.tsNodeIsNull(): return np.getName() +proc printLisp*(gState: State, root: TSNode): string = + var + node = root + nextnode: TSNode + depth = 0 + + while true: + if not node.tsNodeIsNull() and depth > -1: + if gState.pretty: + result &= spaces(depth) + let + (line, col) = gState.getLineCol(node) + result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + else: + break + + if node.tsNodeNamedChildCount() != 0: + if gState.pretty: + result &= "\n" + nextnode = node.tsNodeNamedChild(0) + depth += 1 + else: + if gState.pretty: + result &= ")\n" + else: + result &= ")" + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.tsNodeIsNull(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if gState.pretty: + result &= spaces(depth) & ")\n" + else: + result &= ")" + if node == root: + break + if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +proc printTree*(nimState: NimState, pnode: PNode, offset = "") = + if nimState.gState.debug and pnode.kind != nkNone: + stdout.write "\n# " & offset & $pnode.kind & "(" + case pnode.kind + of nkCharLit: + stdout.write "'" & pnode.intVal.char & "')" + of nkIntLit..nkUInt64Lit: + stdout.write $pnode.intVal & ")" + of nkFloatLit..nkFloat128Lit: + stdout.write $pnode.floatVal & ")" + of nkStrLit..nkTripleStrLit: + stdout.write "\"" & pnode.strVal & "\")" + of nkSym: + stdout.write $pnode.sym & ")" + of nkIdent: + stdout.write "\"" & $pnode.ident.s & "\")" + else: + if pnode.sons.len != 0: + for i in 0 ..< pnode.sons.len: + nimState.printTree(pnode.sons[i], offset & " ") + if i != pnode.sons.len - 1: + stdout.write "," + stdout.write "\n# " & offset & ")" + else: + stdout.write ")" + if offset.len == 0: + echo "" + +proc printDebug*(nimState: NimState, node: TSNode) = + if nimState.gState.debug: + echo nimState.getNodeVal(node).getCommented() + echo nimState.gState.printLisp(node).getCommented() + +proc printDebug*(nimState: NimState, pnode: PNode) = + if nimState.gState.debug: + echo ($pnode).getCommented() + nimState.printTree(pnode) + # Compiler shortcuts proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = diff --git a/nimterop/nim.cfg b/nimterop/nim.cfg deleted file mode 100644 index 913df8c..0000000 --- a/nimterop/nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---path:"$nim" diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 8c5aaa2..26f9cdc 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,55 +4,6 @@ import "."/treesitter/[api, c, cpp] import "."/[ast2, compat, globals, getters] -proc printLisp(gState: State, root: TSNode) = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.tsNodeIsNull() and depth > -1: - if gState.pretty: - stdout.write spaces(depth) - let - (line, col) = gState.getLineCol(node) - stdout.write &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" - else: - break - - if node.tsNodeNamedChildCount() != 0: - if gState.pretty: - echo "" - nextnode = node.tsNodeNamedChild(0) - depth += 1 - else: - if gState.pretty: - echo ")" - else: - stdout.write ")" - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.tsNodeIsNull(): - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - if gState.pretty: - echo spaces(depth) & ")" - else: - stdout.write ")" - if node == root: - break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break - proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -91,7 +42,7 @@ proc process(gState: State, path: string) = tree.tsTreeDelete() if gState.past: - gState.printLisp(root) + echo gState.printLisp(root) elif gState.pnim: gState.printNim(path, root) elif gState.preprocess: From e07983bbe323d3a876420404290725566353c376 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jan 2020 15:59:14 -0600 Subject: [PATCH 03/22] More types, isNil, data in lisp --- nimterop/ast2.nim | 245 ++++++++++++++++++++++++++++++++++++------- nimterop/getters.nim | 65 ++++++++---- 2 files changed, 251 insertions(+), 59 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8790e6f..6f8f543 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -7,6 +7,8 @@ import "."/treesitter/api import "."/[compat, globals, getters] proc addConst(nimState: NimState, node: TSNode) = + # #define X Y + # # (preproc_def # (identifier) # (preproc_arg) @@ -28,35 +30,130 @@ proc addConst(nimState: NimState, node: TSNode) = # If supported literal if val.kind != nkNilLit: - # nnkConstDef( - # nnkPostfix( - # nnkIdent("*"), - # nnkIdent(name) - # ), - # nnkEmpty(), - # nnkXLit(val) + # const X* = Y + # + # nkConstDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkXLit(Y) # ) constDef.add ident constDef.add newNode(nkEmpty) constDef.add val - # nnkConstSection.add + # nkConstSection.add nimState.constSection.add constDef nimState.printDebug(constDef) +proc newTypeIdent(nimState: NimState, node: TSNode): PNode = + # Create nkTypeDef PNode with first ident + let + (name, info) = nimState.getNameInfo(node.getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info) + + result = newNode(nkTypeDef) + result.add ident + result.add newNode(nkEmpty) + +proc newPtrTree(count: int): tuple[parent, child: PNode] = + # Create nkPtrTy tree depending on count + # + # nkPtrTy( + # nkPtrTy( + # .. + # ) + # ) + if count > 0: + result.parent = newNode(nkPtrTy) + result.child = result.parent + for i in 1 ..< count: + let + child = newNode(nkPtrTy) + result.child.add child + result.child = child + +proc addType0(nimState: NimState, node: TSNode) = + let + typeDef = nimState.newTypeIdent(node) + + # type X* = object + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkEmpty() + # ) + # ) + typeDef.add(block: + let + obj = newNode(nkObjectTy) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + + obj + ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addType1(nimState: NimState, node: TSNode) = + let + # node[1] = identifer = name + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = type name + (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) + # TODO - check blank and override + # TODO - use getPtrType() - ptr cchar to cstring + ident = nimState.getIdent(name, info, exported = false) + + # node[1] could have pointers + count = node[1].getPtrCount() + + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + + if count > 0: + let + (parent, child) = newPtrTree(count) + child.add ident + typeDef.add parent + else: + typeDef.add ident + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + proc addType(nimState: NimState, node: TSNode) = # CASE1: # - # typedef struct X Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (type_identifier) - # ) - # + # # CASE 2 # # typedef struct X *Y; @@ -72,20 +169,93 @@ proc addType(nimState: NimState, node: TSNode) = # nimState.printDebug(node) - let - typeDef = newNode(nkTypeDef) - - fdlist = node.inTree("field_declaration_list") - - identNode = if fdlist: node[1] else: node - - (name, info) = nimState.getNameInfo(identNode.getAtom(), nskType) - # TODO - check blank and override - ident = nimState.getIdent(name, info) - - typeNode = - if fdlist: node[0] else: node[1] - + if node.getName() == "struct_specifier": + if node.len == 1: + # struct X; + # + # (struct_specifier + # (type_identifier) + # ) + nimState.addType0(node) + elif node.len == 2: + if node[1].getName() == "field_declaration_list" and node[1].len == 0: + # struct X {}; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + nimState.addType0(node) + elif node.getName() == "type_definition": + if node.len == 2: + let + fdlist = node[0].anyChildInTree("field_declaration_list") + if nimState.getNodeVal(node[1]) == "": + if fdlist.isNil(): + # typedef struct X; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_definition = "") + # ) + nimState.addType0(node) + elif fdlist.len == 0: + # typedef struct X {}; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # (type_definition = "") + # ) + nimState.addType0(node) + else: + let + sspec = node[0].firstChildInTree("struct_specifier") + adecl = node[1].anyChildInTree("array_declarator") + fdecl = node[1].anyChildInTree("function_declarator") + if fdlist.isNil(): + if adecl.isNil and fdecl.isNil: + if not sspec.isNil: + # typedef struct X [*]Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addType1(node) + else: + # typedef struct int Y; + # + # (type_definition + # (type_identifier|primitive_type) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addType1(node) + elif not adecl.isNil: + # typedef X Y[a][..]; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator + # (array_declarator + # (type_identifier) + # (number_literal) + # ) + # ) + # ) + discard proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -100,13 +270,10 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "preproc_def": nimState.addConst(node) of "type_definition": - if node.inTree("struct_specifier"): - nimState.addType(node) - elif node.inTree("enum_specifier"): + if not node.firstChildInTree("enum_specifier").isNil(): nimState.addEnum(node) else: - # Unknown type - result = false + nimState.addType(node) of "struct_specifier": nimState.addType(node) of "enum_specifier": @@ -126,7 +293,7 @@ proc searchTree(nimState: NimState, root: TSNode) = processed = false while true: - if not node.tsNodeIsNull() and depth > -1: + if not node.isNil() and depth > -1: processed = nimState.processNode(node) else: break @@ -137,7 +304,7 @@ proc searchTree(nimState: NimState, root: TSNode) = else: nextnode = node.tsNodeNextNamedSibling() - if nextnode.tsNodeIsNull(): + if nextnode.isNil(): while true: node = node.tsNodeParent() depth -= 1 @@ -145,7 +312,7 @@ proc searchTree(nimState: NimState, root: TSNode) = break if node == root: break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + if not node.tsNodeNextNamedSibling().isNil(): node = node.tsNodeNextNamedSibling() break else: diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 8e03282..ba762f1 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -237,8 +237,11 @@ proc getLit*(str: string): PNode = # TSNode shortcuts +proc isNil*(node: TSNode): bool = + node.tsNodeIsNull() + proc len*(node: TSNode): uint = - if not node.tsNodeIsNull: + if not node.isNil: result = node.tsNodeNamedChildCount().uint proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = @@ -246,51 +249,67 @@ proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = result = node.tsNodeNamedChild(i.uint32) proc getName*(node: TSNode): string {.inline.} = - if not node.tsNodeIsNull: + if not node.isNil: return $node.tsNodeType() +proc getNodeVal*(gState: State, node: TSNode): string = + if not node.isNil: + return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + proc getNodeVal*(nimState: NimState, node: TSNode): string = - if not node.tsNodeIsNull: - return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + nimState.gState.getNodeVal(node) proc getAtom*(node: TSNode): TSNode = - if not node.tsNodeIsNull: + if not node.isNil: # Get child node which is topmost atom if node.getName() in gAtoms: return node elif node.len() != 0: return node[0].getAtom() -proc getPtrCount*(node: TSNode): string = - if not node.tsNodeIsNull: +proc getPtrCount*(node: TSNode): int = + if not node.isNil: # Get number of ptr nodes in tree var cnode = node while "pointer_declarator" in cnode.getName(): - result &= "ptr " + result += 1 if cnode.len() != 0: cnode = cnode[0] else: break proc getDeclarator*(node: TSNode): TSNode = - if not node.tsNodeIsNull: + if not node.isNil: # Return if child is a function or array declarator if node.getName() in ["function_declarator", "array_declarator"]: return node elif node.len() != 0: return node[0].getDeclarator() -proc inTree*(node: TSNode, ntype: string): bool = +proc firstChildInTree*(node: TSNode, ntype: string): TSNode = # Search for node type in tree - first children - result = false var cnode = node - while not cnode.tsNodeIsNull: + while not cnode.isNil: if cnode.getName() == ntype: - return true + return cnode cnode = cnode[0] +proc anyChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type anywhere in tree - depth first + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + for i in 0 ..< cnode.len: + let + ccnode = cnode[i].anyChildInTree(ntype) + if not ccnode.isNil(): + return ccnode + cnode = cnode.tsNodeNextNamedSibling() + proc inChildren*(node: TSNode, ntype: string): bool = # Search for node type in immediate children result = false @@ -320,11 +339,11 @@ proc getPxName*(node: TSNode, offset: int): string = np = node count = 0 - while not np.tsNodeIsNull() and count < offset: + while not np.isNil() and count < offset: np = np.tsNodeParent() count += 1 - if count == offset and not np.tsNodeIsNull(): + if count == offset and not np.isNil(): return np.getName() proc printLisp*(gState: State, root: TSNode): string = @@ -334,12 +353,16 @@ proc printLisp*(gState: State, root: TSNode): string = depth = 0 while true: - if not node.tsNodeIsNull() and depth > -1: + if not node.isNil() and depth > -1: if gState.pretty: result &= spaces(depth) let (line, col) = gState.getLineCol(node) result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + let + val = gState.getNodeVal(node) + if "\n" notin val and " " notin val: + result &= &" \"{val}\"" else: break @@ -355,7 +378,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= ")" nextnode = node.tsNodeNextNamedSibling() - if nextnode.tsNodeIsNull(): + if nextnode.isNil(): while true: node = node.tsNodeParent() depth -= 1 @@ -367,7 +390,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= ")" if node == root: break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + if not node.tsNodeNextNamedSibling().isNil(): node = node.tsNodeNextNamedSibling() break else: @@ -409,12 +432,12 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: - echo nimState.getNodeVal(node).getCommented() + echo ("Input => " & nimState.getNodeVal(node)).getCommented() echo nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: - echo ($pnode).getCommented() + echo ("Output => " & $pnode).getCommented() nimState.printTree(pnode) # Compiler shortcuts @@ -445,6 +468,8 @@ proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = " let name = nimState.getNodeVal(node) result.name = nimState.getIdentifier(name, kind, parent) + if kind == nskType: + result.name = result.name.getType() result.info = nimState.getLineInfo(node) proc getCurrentHeader*(fullpath: string): string = From 0965eea3e97e64d8993a5e97387a173a4f581699 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jan 2020 18:24:21 -0600 Subject: [PATCH 04/22] ptr cchar to cstring, array support, named type procs --- nimterop/ast2.nim | 200 +++++++++++++++++++++++++++++++------------ nimterop/getters.nim | 29 +++++-- 2 files changed, 163 insertions(+), 66 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6f8f543..dc1a4b1 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -60,24 +60,51 @@ proc newTypeIdent(nimState: NimState, node: TSNode): PNode = result.add ident result.add newNode(nkEmpty) -proc newPtrTree(count: int): tuple[parent, child: PNode] = +proc newPtrTree(count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # + # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring + # # nkPtrTy( # nkPtrTy( - # .. + # typ # ) # ) + var + count = count + if typ.kind == nkIdent: + let + tname = typ.ident.s + ptname = getPtrType(tname) + if tname != ptname: + typ.ident.s = ptname + count -= 1 if count > 0: - result.parent = newNode(nkPtrTy) - result.child = result.parent + result = newNode(nkPtrTy) + var + parent = result + child: PNode for i in 1 ..< count: - let - child = newNode(nkPtrTy) - result.child.add child - result.child = child + child = newNode(nkPtrTy) + parent.add child + parent = child + parent.add typ + else: + result = typ -proc addType0(nimState: NimState, node: TSNode) = +proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = + result = newNode(nkBracketExpr) + + let + (_, info) = nimState.getNameInfo(node, nskType) + ident = nimState.getIdent("array", info, exported = false) + + result.add ident + result.add size + result.add typ + +proc addTypeObject(nimState: NimState, node: TSNode) = + # Add a type of object let typeDef = nimState.newTypeIdent(node) @@ -110,7 +137,8 @@ proc addType0(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc addType1(nimState: NimState, node: TSNode) = +proc addTypeTyped(nimState: NimState, node: TSNode) = + # Add a type of a specific type let # node[1] = identifer = name typeDef = nimState.newTypeIdent(node[1]) @@ -118,10 +146,9 @@ proc addType1(nimState: NimState, node: TSNode) = # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) # TODO - check blank and override - # TODO - use getPtrType() - ptr cchar to cstring ident = nimState.getIdent(name, info, exported = false) - # node[1] could have pointers + # node[1] could have nested pointers count = node[1].getPtrCount() # type X* = [ptr ..] Y @@ -137,13 +164,62 @@ proc addType1(nimState: NimState, node: TSNode) = # ) # ) - if count > 0: + # Skip typedef X X; + if $typeDef[0][1] != name: + if count > 0: + # If pointers + typeDef.add newPtrTree(count, ident) + else: + typeDef.add ident + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addTypeArray(nimState: NimState, node: TSNode) = + # Add a type of a array type + let + # node[1] = identifer = name + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = type name + (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info, exported = false) + + # Top-most array declarator + adecl = node[1].firstChildInTree("array_declarator") + + # node[1] could have nested arrays + acount = adecl.getArrayCount() + innermost = adecl.mostNestedChildInTree() + + # node[1] could have nested pointers - type + tcount = node[1].getPtrCount() + + # Name could have nested pointers + ncount = innermost[0].getPtrCount() + + var + cnode = adecl + typ = ident + + if tcount > 0: + # If pointers + typ = newPtrTree(tcount, typ) + + for i in 0 ..< acount: let - (parent, child) = newPtrTree(count) - child.add ident - typeDef.add parent - else: - typeDef.add ident + size = nimState.getNodeVal(cnode[1]).getLit() + if size.kind != nkNilLit: + typ = nimState.newArrayTree(cnode, typ, size) + cnode = cnode[0] + + if ncount > 0: + typ = newPtrTree(ncount, typ) + + typeDef.add typ # nkTypeSection.add nimState.typeSection.add typeDef @@ -151,22 +227,6 @@ proc addType1(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = - # CASE1: - # - # - # CASE 2 - # - # typedef struct X *Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - # (type_identifier) - # ) - # ) - # nimState.printDebug(node) if node.getName() == "struct_specifier": @@ -176,7 +236,7 @@ proc addType(nimState: NimState, node: TSNode) = # (struct_specifier # (type_identifier) # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif node.len == 2: if node[1].getName() == "field_declaration_list" and node[1].len == 0: # struct X {}; @@ -185,7 +245,7 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # (field_declaration_list = "{}") # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif node.getName() == "type_definition": if node.len == 2: let @@ -200,7 +260,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif fdlist.len == 0: # typedef struct X {}; # @@ -211,16 +271,17 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) - nimState.addType0(node) + nimState.addTypeObject(node) else: let sspec = node[0].firstChildInTree("struct_specifier") - adecl = node[1].anyChildInTree("array_declarator") fdecl = node[1].anyChildInTree("function_declarator") + adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): if adecl.isNil and fdecl.isNil: if not sspec.isNil: - # typedef struct X [*]Y; + # typedef struct X Y; + # typedef struct X *Y; # # (type_definition # (struct_specifier @@ -230,9 +291,10 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) - nimState.addType1(node) + nimState.addTypeTyped(node) else: - # typedef struct int Y; + # typedef X Y; + # typedef X *Y; # # (type_definition # (type_identifier|primitive_type) @@ -240,22 +302,46 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) - nimState.addType1(node) - elif not adecl.isNil: - # typedef X Y[a][..]; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - # (array_declarator - # (type_identifier) - # (number_literal) - # ) - # ) - # ) + nimState.addTypeTyped(node) + elif not fdecl.isNil: discard + elif not adecl.isNil: + if not sspec.isNil: + # typedef struct X Y[a][..]; + # typedef struct X *Y[a][..]; + # typedef struct X *(*Y)[a][..]; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) + else: + # typedef X Y[a][..]; + # typedef X *Y[a][..]; + # typedef X *(*Y)[a][..]; + # + # (type_definition + # (type_identifier|primitive_type) + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index ba762f1..31dd850 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -97,14 +97,10 @@ proc getType*(str: string): string = proc getPtrType*(str: string): string = result = case str: - of "ptr cchar": + of "cchar": "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": + of "object": "pointer" - of "ptr ptr object": - "ptr pointer" else: str @@ -267,18 +263,24 @@ proc getAtom*(node: TSNode): TSNode = elif node.len() != 0: return node[0].getAtom() -proc getPtrCount*(node: TSNode): int = +proc getXCount*(node: TSNode, ntype: string): int = if not node.isNil: - # Get number of ptr nodes in tree + # Get number of ntype nodes nested in tree var cnode = node - while "pointer_declarator" in cnode.getName(): + while ntype in cnode.getName(): result += 1 if cnode.len() != 0: cnode = cnode[0] else: break +proc getPtrCount*(node: TSNode): int = + node.getXCount("pointer_declarator") + +proc getArrayCount*(node: TSNode): int = + node.getXCount("array_declarator") + proc getDeclarator*(node: TSNode): TSNode = if not node.isNil: # Return if child is a function or array declarator @@ -310,6 +312,15 @@ proc anyChildInTree*(node: TSNode, ntype: string): TSNode = return ccnode cnode = cnode.tsNodeNextNamedSibling() +proc mostNestedChildInTree*(node: TSNode): TSNode = + # Search for the most nested child of node's type in tree + var + cnode = node + ntype = cnode.getName() + while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype: + cnode = cnode[0] + result = cnode + proc inChildren*(node: TSNode, ntype: string): bool = # Search for node type in immediate children result = false From 27e30e30f72780bfcfcc7960d3cdc2ab929d11fc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 4 Jan 2020 11:55:15 -0600 Subject: [PATCH 05/22] Upgrade tree-sitter version, collapse duplicate if, begin func type --- nimterop/ast2.nim | 153 ++++++++++++++++++++++++++------------------- nimterop/setup.nim | 6 +- 2 files changed, 92 insertions(+), 67 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dc1a4b1..9f64eef 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -106,6 +106,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = proc addTypeObject(nimState: NimState, node: TSNode) = # Add a type of object let + # TODO - check blank and override typeDef = nimState.newTypeIdent(node) # type X* = object @@ -141,11 +142,11 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # Add a type of a specific type let # node[1] = identifer = name + # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - # TODO - check blank and override ident = nimState.getIdent(name, info, exported = false) # node[1] could have nested pointers @@ -178,14 +179,14 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addTypeArray(nimState: NimState, node: TSNode) = - # Add a type of a array type + # Add a type of array type let # node[1] = identifer = name + # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - # TODO - check blank and override ident = nimState.getIdent(name, info, exported = false) # Top-most array declarator @@ -226,6 +227,27 @@ proc addTypeArray(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) +proc addTypeFunc(nimState: NimState, node: TSNode) = + # Add a type of function type + let + # node[1] = identifier = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = return type name + (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + + # node[1] could have nested pointers + count = node[1].getArrayCount() + + # Parameter list + plist = node[1].anyChildInTree("parameter_list") + + var + retType = nimState.getIdent(rname, rinfo) + if count > 0: + retType = newPtrTree(count, retType) + proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -279,69 +301,72 @@ proc addType(nimState: NimState, node: TSNode) = adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): if adecl.isNil and fdecl.isNil: - if not sspec.isNil: - # typedef struct X Y; - # typedef struct X *Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # ) - nimState.addTypeTyped(node) - else: - # typedef X Y; - # typedef X *Y; - # - # (type_definition - # (type_identifier|primitive_type) - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # ) - nimState.addTypeTyped(node) + # typedef X Y; + # typedef X *Y; + # typedef struct X Y; + # typedef struct X *Y; + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addTypeTyped(node) elif not fdecl.isNil: - discard + # typedef X (*Y)(a1, a2, a3); + # typedef X *(*Y)(a1, a2, a3); + # typedef struct X (*Y)(a1, a2, a3); + # typedef struct X *(*Y)(a1, a2, a3); + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator + # (type_identifer) + # ) + # ) + # (parameter_list + # (parameter_declaration + # (struct_specifier|type_identifier|primitive_type|array_declarator|function_declarator) + # (identifier - optional) + # ) + # ) + # ) + # ) + # ) + nimState.addTypeFunc(node) elif not adecl.isNil: - if not sspec.isNil: - # typedef struct X Y[a][..]; - # typedef struct X *Y[a][..]; - # typedef struct X *(*Y)[a][..]; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - optional, nested - # (array_declarator - nested - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # (number_literal) - # ) - # ) - # ) - nimState.addTypeArray(node) - else: - # typedef X Y[a][..]; - # typedef X *Y[a][..]; - # typedef X *(*Y)[a][..]; - # - # (type_definition - # (type_identifier|primitive_type) - # (pointer_declarator - optional, nested - # (array_declarator - nested - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # (number_literal) - # ) - # ) - # ) - nimState.addTypeArray(node) + # typedef struct X Y[a][..]; + # typedef struct X *Y[a][..]; + # typedef struct X *(*Y)[a][..]; + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 3f16dd7..cfa9a63 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -6,7 +6,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.5") +""", "0.16.2") gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ *.c @@ -36,7 +36,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.16.0") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -48,7 +48,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.16.0") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); From 52e82bb2b44475c993a15991c1f209153e8399fe Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Jan 2020 01:51:15 -0600 Subject: [PATCH 06/22] Add proc type, reverse X count --- nimterop/ast2.nim | 262 ++++++++++++++++++++++++++++++++++++------- nimterop/getters.nim | 15 ++- 2 files changed, 232 insertions(+), 45 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 9f64eef..68956ae 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -93,6 +93,13 @@ proc newPtrTree(count: int, typ: PNode): PNode = result = typ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = + # Create nkBracketExpr tree depending on input + # + # nkBracketExpr( + # nkIdent("array"), + # size, + # typ + # ) result = newNode(nkBracketExpr) let @@ -103,6 +110,86 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add size result.add typ +proc getTypeArray(nimState: NimState, node: TSNode): PNode +proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode + +proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64): PNode = + # Create nkIdentDefs tree for specified parameter + result = newNode(nkIdentDefs) + + let + # node[0] - param type + (tname, tinfo) = nimState.getNameInfo(node[0], nskType) + tident = nimState.getIdent(tname, tinfo, exported = false) + + if node.len == 1: + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported = false) + result.add pident + result.add tident + result.add newNode(nkEmpty) + else: + let + fdecl = node[1].anyChildInTree("function_declarator") + adecl = node[1].anyChildInTree("array_declarator") + abst = node[1].getName() == "abstract_pointer_declarator" + if fdecl.isNil and adecl.isNil: + if abst: + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported = false) + acount = node[1].getXCount("abstract_pointer_declarator") + result.add pident + result.add newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + + count = node[1].getPtrCount() + result.add pident + if count > 0: + result.add newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + result.add pident + result.add nimState.getTypeProc(node, name) + result.add newNode(nkEmpty) + elif not adecl.isNil: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + result.add pident + result.add nimState.getTypeArray(node) + result.add newNode(nkEmpty) + else: + result = nil + +proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + result = newNode(nkProcTy) + + let + fparam = newNode(nkFormalParams) + result.add fparam + result.add newNode(nkEmpty) + + # Add return type + fparam.add rtyp + + if not node.isNil: + for i in 0 ..< node.len: + let + a = nimState.newProcParam(name, node[i], i) + if not a.isNil: + fparam.add a + proc addTypeObject(nimState: NimState, node: TSNode) = # Add a type of object let @@ -152,19 +239,6 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # node[1] could have nested pointers count = node[1].getPtrCount() - # type X* = [ptr ..] Y - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkIdent("Y") - # ) - # ) - # Skip typedef X X; if $typeDef[0][1] != name: if count > 0: @@ -173,18 +247,27 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = else: typeDef.add ident + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) -proc addTypeArray(nimState: NimState, node: TSNode) = - # Add a type of array type +proc getTypeArray(nimState: NimState, node: TSNode): PNode = + # Create array type tree let - # node[1] = identifer = name - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) - # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) ident = nimState.getIdent(name, info, exported = false) @@ -199,54 +282,156 @@ proc addTypeArray(nimState: NimState, node: TSNode) = # node[1] could have nested pointers - type tcount = node[1].getPtrCount() - # Name could have nested pointers - ncount = innermost[0].getPtrCount() + # Name could be nested pointer to array + # + # (.. + # (array_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = innermost[0].getAtom().tsNodeParent().getPtrCount(reverse = true) + result = ident var cnode = adecl - typ = ident if tcount > 0: # If pointers - typ = newPtrTree(tcount, typ) + result = newPtrTree(tcount, result) for i in 0 ..< acount: let size = nimState.getNodeVal(cnode[1]).getLit() if size.kind != nkNilLit: - typ = nimState.newArrayTree(cnode, typ, size) + result = nimState.newArrayTree(cnode, result, size) cnode = cnode[0] if ncount > 0: - typ = newPtrTree(ncount, typ) + result = newPtrTree(ncount, result) + +proc addTypeArray(nimState: NimState, node: TSNode) = + # Add a type of array type + let + # node[1] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + + typ = nimState.getTypeArray(node) typeDef.add typ + # type X* = [ptr] array[x, [ptr] Y] + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkBracketExpr( + # nkIdent("array") + # nkXLit(x), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # ) + # ) + # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) -proc addTypeFunc(nimState: NimState, node: TSNode) = - # Add a type of function type +proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode = + # Create proc type tree let - # node[1] = identifier = name - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) - # node[0] = identifier = return type name (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) - # node[1] could have nested pointers - count = node[1].getArrayCount() - # Parameter list plist = node[1].anyChildInTree("parameter_list") + # node[1] could have nested pointers + tcount = node[1].getPtrCount() + + # Name could be nested pointer to function + # + # (.. + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = node[1].getAtom().tsNodeParent().getPtrCount(reverse = true) + + # Return type var - retType = nimState.getIdent(rname, rinfo) - if count > 0: - retType = newPtrTree(count, retType) + retType = nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = newPtrTree(tcount, retType) + + # Proc with return type and params + result = nimState.newProcTree(name, plist, retType) + if ncount > 1: + result = newPtrTree(ncount-1, result) + +proc addTypeProc(nimState: NimState, node: TSNode) = + # Add a type of proc type + let + # node[1] = identifier = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + name = $typeDef[0][1] + + procTy = nimState.getTypeProc(node, name) + + typeDef.add procTy + + # type X* = proc(a1: Y, a2: Z): P + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkProcTy( + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkEmpty() + # ) + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -296,7 +481,6 @@ proc addType(nimState: NimState, node: TSNode) = nimState.addTypeObject(node) else: let - sspec = node[0].firstChildInTree("struct_specifier") fdecl = node[1].anyChildInTree("function_declarator") adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): @@ -345,7 +529,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) - nimState.addTypeFunc(node) + nimState.addTypeProc(node) elif not adecl.isNil: # typedef struct X Y[a][..]; # typedef struct X *Y[a][..]; diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 31dd850..c25ce3c 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -263,22 +263,25 @@ proc getAtom*(node: TSNode): TSNode = elif node.len() != 0: return node[0].getAtom() -proc getXCount*(node: TSNode, ntype: string): int = +proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if not node.isNil: # Get number of ntype nodes nested in tree var cnode = node while ntype in cnode.getName(): result += 1 - if cnode.len() != 0: - cnode = cnode[0] + if reverse: + cnode = cnode.tsNodeParent() else: - break + if cnode.len() != 0: + cnode = cnode[0] + else: + break -proc getPtrCount*(node: TSNode): int = +proc getPtrCount*(node: TSNode, reverse = false): int = node.getXCount("pointer_declarator") -proc getArrayCount*(node: TSNode): int = +proc getArrayCount*(node: TSNode, reverse = false): int = node.getXCount("array_declarator") proc getDeclarator*(node: TSNode): TSNode = From 48bde874e51035b6b85b8adac63e77fbcb31bf71 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 7 Jan 2020 01:12:42 -0600 Subject: [PATCH 07/22] Struct fields support, multiple bugs --- config.nims | 5 + nimterop/ast2.nim | 366 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 264 insertions(+), 107 deletions(-) diff --git a/config.nims b/config.nims index d312c07..e3d4ce0 100644 --- a/config.nims +++ b/config.nims @@ -13,3 +13,8 @@ switch("stackTrace", "on") # Path to compiler switch("path", "$nim") + +# Case objects +when not defined(danger): + switch("define", "nimOldCaseObjects") + diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 68956ae..caa2a13 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -49,12 +49,16 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) -proc newTypeIdent(nimState: NimState, node: TSNode): PNode = +proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = # Create nkTypeDef PNode with first ident let (name, info) = nimState.getNameInfo(node.getAtom(), nskType) # TODO - check blank and override - ident = nimState.getIdent(name, info) + ident = + if override.len != 0: + nimState.getIdent(override, info) + else: + nimState.getIdent(name, info) result = newNode(nkTypeDef) result.add ident @@ -111,10 +115,29 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add typ proc getTypeArray(nimState: NimState, node: TSNode): PNode -proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode -proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64): PNode = - # Create nkIdentDefs tree for specified parameter +proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64, exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter or object field + # + # For proc, param should not be exported + # + # nkIdentDefs( + # nkIdent(pname), + # typ, + # nkEmpty() + # ) + # + # For object, field should be exported + # + # nkIdentDefs( + # nkPostfix( + # nkIdent("*"), + # nkIdent(pname) + # ), + # typ, + # nkEmpty() + # ) result = newNode(nkIdentDefs) let @@ -123,9 +146,12 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 tident = nimState.getIdent(tname, tinfo, exported = false) if node.len == 1: + # Only for proc with no named param - create a param name based on offset + # + # int func(char, int); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported = false) + pident = nimState.getIdent(pname, tinfo, exported) result.add pident result.add tident result.add newNode(nkEmpty) @@ -136,17 +162,22 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 abst = node[1].getName() == "abstract_pointer_declarator" if fdecl.isNil and adecl.isNil: if abst: + # Only for proc with no named param with pointer type + # Create a param name based on offset + # + # int func(char *, int **); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported = false) + pident = nimState.getIdent(pname, tinfo, exported) acount = node[1].getXCount("abstract_pointer_declarator") result.add pident result.add newPtrTree(acount, tident) result.add newNode(nkEmpty) else: + # Named param, simple type let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) count = node[1].getPtrCount() result.add pident @@ -156,16 +187,18 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 result.add tident result.add newNode(nkEmpty) elif not fdecl.isNil: + # Named param, function pointer let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeProc(node, name) + result.add nimState.getTypeProc(name, node) result.add newNode(nkEmpty) elif not adecl.isNil: + # Named param, array type let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeArray(node) result.add newNode(nkEmpty) @@ -173,81 +206,66 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 result = nil proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + # Create nkProcTy tree for specified proc type + # + # nkProcTy( + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ), + # nkEmpty() + # ) result = newNode(nkProcTy) let fparam = newNode(nkFormalParams) - result.add fparam - result.add newNode(nkEmpty) # Add return type fparam.add rtyp if not node.isNil: for i in 0 ..< node.len: + # Add nkIdentDefs for each param let - a = nimState.newProcParam(name, node[i], i) - if not a.isNil: - fparam.add a + param = nimState.newIdentDefs(name, node[i], i, exported = false) + if not param.isNil: + fparam.add param -proc addTypeObject(nimState: NimState, node: TSNode) = - # Add a type of object - let - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node) + result.add fparam + result.add newNode(nkEmpty) - # type X* = object +proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = + # Create nkRecList tree for specified object # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkObjectTy( - # nkEmpty(), - # nkEmpty(), - # nkEmpty() + # nkRecList( + # nkIdentDefs( # multiple depending on fields + # .. # ) # ) - typeDef.add(block: - let - obj = newNode(nkObjectTy) - obj.add newNode(nkEmpty) - obj.add newNode(nkEmpty) - obj.add newNode(nkEmpty) + if not node.isNil: + result = newNode(nkRecList) - obj - ) + for i in 0 ..< node.len: + # Add nkIdentDefs for each field + let + field = nimState.newIdentDefs(name, node[i], i, exported = true) + if not field.isNil: + result.add field - # nkTypeSection.add - nimState.typeSection.add typeDef - - nimState.printDebug(typeDef) - -proc addTypeTyped(nimState: NimState, node: TSNode) = - # Add a type of a specific type +proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "") = + # Add a type of object + # + # If `override` is set, use it as the name + # If `duplicate` is set, don't add the same name let - # node[1] = identifer = name # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) + typeDef = nimState.newTypeIdent(node, override) + name = $typeDef[0][1] - # node[0] = identifier = type name - (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - ident = nimState.getIdent(name, info, exported = false) - - # node[1] could have nested pointers - count = node[1].getPtrCount() - - # Skip typedef X X; - if $typeDef[0][1] != name: - if count > 0: - # If pointers - typeDef.add newPtrTree(count, ident) - else: - typeDef.add ident - - # type X* = [ptr ..] Y + if name != duplicate: + # type X* = object # # nkTypeDef( # nkPostfix( @@ -255,16 +273,83 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # nkIdent("X") # ), # nkEmpty(), - # nkPtrTy( # optional, nested - # nkIdent("Y") + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkEmpty() # ) # ) + let + obj = newNode(nkObjectTy) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + + let + fdlist = node.anyChildInTree("field_declaration_list") + if not fdlist.isNil and fdlist.len > 0: + # Add fields to object if present + obj.add nimState.newRecListTree(name, fdlist) + else: + obj.add newNode(nkEmpty) + + typeDef.add obj # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) +proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = "") = + # Add a type of a specified type + # + # If `toverride` is set, use it as the type name + # If `duplicate` is set, don't add the same name + for i in 1 ..< node.len: + # Add a type of a specific type + let + # node[i] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[i]) + + # node[0] = identifier = type name + (tname0, tinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + + # Override type name + tname = if toverride.len != 0: toverride else: tname0 + + ident = nimState.getIdent(tname, tinfo, exported = false) + + # node[i] could have nested pointers + count = node[i].getPtrCount() + + # Skip typedef X X; + if $typeDef[0][1] != tname: + if count > 0: + # If pointers + typeDef.add newPtrTree(count, ident) + else: + typeDef.add ident + + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + else: + nimState.addTypeObject(node, duplicate) + proc getTypeArray(nimState: NimState, node: TSNode): PNode = # Create array type tree let @@ -350,7 +435,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode = +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # Create proc type tree let # node[0] = identifier = return type name @@ -396,7 +481,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = typeDef = nimState.newTypeIdent(node[1]) name = $typeDef[0][1] - procTy = nimState.getTypeProc(node, name) + procTy = nimState.getTypeProc(name, node) typeDef.add procTy @@ -437,48 +522,59 @@ proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) if node.getName() == "struct_specifier": - if node.len == 1: - # struct X; - # - # (struct_specifier - # (type_identifier) - # ) - nimState.addTypeObject(node) - elif node.len == 2: - if node[1].getName() == "field_declaration_list" and node[1].len == 0: - # struct X {}; - # - # (struct_specifier - # (type_identifier) - # (field_declaration_list = "{}") - # ) - nimState.addTypeObject(node) + # struct X; + # + # (struct_specifier + # (type_identifier) + # ) + # + # struct X {}; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # + # struct X { char *a1; }; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list + # (field_declaration + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (field_identifier) + # ) + # (field_declaration ...) + # ) + nimState.addTypeObject(node) elif node.getName() == "type_definition": - if node.len == 2: + if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") if nimState.getNodeVal(node[1]) == "": - if fdlist.isNil(): - # typedef struct X; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (type_definition = "") - # ) - nimState.addTypeObject(node) - elif fdlist.len == 0: - # typedef struct X {}; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # (field_declaration_list = "{}") - # ) - # (type_definition = "") - # ) - nimState.addTypeObject(node) + # typedef struct X; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_definition = "") + # ) + # + # typedef struct X {}; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # (type_definition = "") + # ) + nimState.addTypeObject(node[0]) else: let fdecl = node[1].anyChildInTree("function_declarator") @@ -551,6 +647,62 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) nimState.addTypeArray(node) + else: + if node.firstChildInTree("field_declaration_list").isNil: + # typedef struct X { .. } Y, *Z; + # + # (type_definition + # (struct_specifier + # (type_identifier) - named struct <==== + # (field_declaration_list + # (field_declaration - optional, multiple + # (type_identifier|primitive_type|) + # (function_declarator|array_declarator + # .. + # ) + # + # (field_identifier) + # ) + # ) + # ) + # + # (type_identifier) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + + # First add struct as object + nimState.addTypeObject(node[0]) + + if node.len > 1 and nimState.getNodeVal(node[1]) != "": + # Add any additional names + nimState.addTypeTyped(node, duplicate = nimState.getNodeVal(node[0].getAtom())) + else: + # Same as above except unnamed struct + # + # typedef struct { .. } Y, *Z; + + # Get any name that isn't a pointer + let + name = block: + var + name = "" + for i in 1 ..< node.len: + if node[i].getName() == "type_identifier": + name = nimState.getNodeVal(node[i].getAtom()) + + if name.len == 0: + name = nimState.getUniqueIdentifier("STRUCT") + + name + + # Now add struct as object with specified name + nimState.addTypeObject(node[0], override = name) + + if node.len > 1 and nimState.getNodeVal(node[1]) != "": + # Add any additional names except duplicate + nimState.addTypeTyped(node, toverride = name, duplicate = name) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) From 49607db05b931f13dd99eeb4d7ab2d54e223c4f9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 9 Jan 2020 13:58:34 -0600 Subject: [PATCH 08/22] Fix type bugs, enable basic tests --- nimterop.nimble | 23 ++++---- nimterop/ast2.nim | 128 +++++++++++++++++++++++++++--------------- nimterop/getters.nim | 9 ++- nimterop/globals.nim | 10 ++-- tests/include/test3.h | 42 ++++++++++++++ tests/tnimterop2.nim | 71 +++++++++++++++++++++++ 6 files changed, 223 insertions(+), 60 deletions(-) create mode 100644 tests/include/test3.h create mode 100644 tests/tnimterop2.nim diff --git a/nimterop.nimble b/nimterop.nimble index 101966e..86be4dc 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -36,19 +36,22 @@ task docs, "Generate docs": task test, "Test": buildToastTask() - execTest "tests/tnimterop_c.nim" - execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" - execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" - execTest "tests/tpcre.nim" + execTest "tests/tnimterop2.nim" + + # Commented out until newalgo is ready + # execTest "tests/tnimterop_c.nim" + # execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" + # execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" + # execTest "tests/tpcre.nim" # Platform specific tests - when defined(Windows): - execTest "tests/tmath.nim" - if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): - execTest "tests/tsoloud.nim" + # when defined(Windows): + # execTest "tests/tmath.nim" + # if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): + # execTest "tests/tsoloud.nim" # getHeader tests - withDir("tests"): - execCmd("nim e getheader.nims") + # withDir("tests"): + # execCmd("nim e getheader.nims") docsTask() diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index caa2a13..6c62b05 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -13,6 +13,7 @@ proc addConst(nimState: NimState, node: TSNode) = # (identifier) # (preproc_arg) # ) + decho("addConst()") nimState.printDebug(node) if node[0].getName() == "identifier" and @@ -51,6 +52,8 @@ proc addConst(nimState: NimState, node: TSNode) = proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = # Create nkTypeDef PNode with first ident + # + # If `override`, use it instead of node.getAtom() for name let (name, info) = nimState.getNameInfo(node.getAtom(), nskType) # TODO - check blank and override @@ -60,30 +63,46 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = else: nimState.getIdent(name, info) + # type name* = + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkEmpty() + # ) result = newNode(nkTypeDef) result.add ident result.add newNode(nkEmpty) -proc newPtrTree(count: int, typ: PNode): PNode = +proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring - # - # nkPtrTy( - # nkPtrTy( - # typ - # ) - # ) var count = count + chng = false if typ.kind == nkIdent: let tname = typ.ident.s ptname = getPtrType(tname) if tname != ptname: - typ.ident.s = ptname + # If Nim type available, use that ident + result = nimState.getIdent(ptname, typ.info, exported = false) + chng = true + # One ptr reduced count -= 1 if count > 0: + # Nested nkPtrTy(typ) depending on count + # + # [ptr ...] typ + # + # nkPtrTy( + # nkPtrTy( + # typ + # ) + # ) result = newNode(nkPtrTy) var parent = result @@ -93,11 +112,17 @@ proc newPtrTree(count: int, typ: PNode): PNode = parent.add child parent = child parent.add typ - else: + elif not chng: + # Either no ptr, or none left after Nim type adjustment result = typ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # Create nkBracketExpr tree depending on input + let + (_, info) = nimState.getNameInfo(node, nskType) + ident = nimState.getIdent("array", info, exported = false) + + # array[size, typ] # # nkBracketExpr( # nkIdent("array"), @@ -105,11 +130,6 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # typ # ) result = newNode(nkBracketExpr) - - let - (_, info) = nimState.getNameInfo(node, nskType) - ident = nimState.getIdent("array", info, exported = false) - result.add ident result.add size result.add typ @@ -122,6 +142,8 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 # # For proc, param should not be exported # + # pname: [ptr ..] typ + # # nkIdentDefs( # nkIdent(pname), # typ, @@ -130,6 +152,8 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 # # For object, field should be exported # + # pname*: [ptr ..] typ + # # nkIdentDefs( # nkPostfix( # nkIdent("*"), @@ -171,7 +195,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 pident = nimState.getIdent(pname, tinfo, exported) acount = node[1].getXCount("abstract_pointer_declarator") result.add pident - result.add newPtrTree(acount, tident) + result.add nimState.newPtrTree(acount, tident) result.add newNode(nkEmpty) else: # Named param, simple type @@ -182,7 +206,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 count = node[1].getPtrCount() result.add pident if count > 0: - result.add newPtrTree(count, tident) + result.add nimState.newPtrTree(count, tident) else: result.add tident result.add newNode(nkEmpty) @@ -207,18 +231,6 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkProcTy tree for specified proc type - # - # nkProcTy( - # nkFormalParams( - # rtyp, - # nkIdentDefs( # multiple depending on params - # .. - # ) - # ), - # nkEmpty() - # ) - result = newNode(nkProcTy) - let fparam = newNode(nkFormalParams) @@ -233,18 +245,32 @@ proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): P if not param.isNil: fparam.add param + # proc(pname: ptyp ..): rtyp + # + # nkProcTy( + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ), + # nkEmpty() + # ) + result = newNode(nkProcTy) result.add fparam result.add newNode(nkEmpty) proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object - # - # nkRecList( - # nkIdentDefs( # multiple depending on fields - # .. - # ) - # ) if not node.isNil: + # fname*: ftyp + # .. + # + # nkRecList( + # nkIdentDefs( # multiple depending on fields + # .. + # ) + # ) result = newNode(nkRecList) for i in 0 ..< node.len: @@ -259,6 +285,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = # # If `override` is set, use it as the name # If `duplicate` is set, don't add the same name + decho("addTypeObject()") let # TODO - check blank and override typeDef = nimState.newTypeIdent(node, override) @@ -304,6 +331,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = # # If `toverride` is set, use it as the type name # If `duplicate` is set, don't add the same name + decho("addTypeTyped()") for i in 1 ..< node.len: # Add a type of a specific type let @@ -326,7 +354,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = if $typeDef[0][1] != tname: if count > 0: # If pointers - typeDef.add newPtrTree(count, ident) + typeDef.add nimState.newPtrTree(count, ident) else: typeDef.add ident @@ -348,7 +376,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = nimState.printDebug(typeDef) else: - nimState.addTypeObject(node, duplicate) + nimState.addTypeObject(node, duplicate = duplicate) proc getTypeArray(nimState: NimState, node: TSNode): PNode = # Create array type tree @@ -388,7 +416,7 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = if tcount > 0: # If pointers - result = newPtrTree(tcount, result) + result = nimState.newPtrTree(tcount, result) for i in 0 ..< acount: let @@ -398,10 +426,11 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = cnode = cnode[0] if ncount > 0: - result = newPtrTree(ncount, result) + result = nimState.newPtrTree(ncount, result) proc addTypeArray(nimState: NimState, node: TSNode) = # Add a type of array type + decho("addTypeArray()") let # node[1] = identifer = name # TODO - check blank and override @@ -466,15 +495,16 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = var retType = nimState.getIdent(rname, rinfo, exported = false) if tcount > 0: - retType = newPtrTree(tcount, retType) + retType = nimState.newPtrTree(tcount, retType) # Proc with return type and params result = nimState.newProcTree(name, plist, retType) if ncount > 1: - result = newPtrTree(ncount-1, result) + result = nimState.newPtrTree(ncount-1, result) proc addTypeProc(nimState: NimState, node: TSNode) = # Add a type of proc type + decho("addTypeProc()") let # node[1] = identifier = name # TODO - check blank and override @@ -519,6 +549,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = + decho("addType()") nimState.printDebug(node) if node.getName() == "struct_specifier": @@ -550,12 +581,14 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (field_declaration ...) # ) + decho("addType(): case 1") nimState.addTypeObject(node) elif node.getName() == "type_definition": if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") - if nimState.getNodeVal(node[1]) == "": + if (fdlist.isNil or (not fdlist.isNil and fdlist.len == 0)) and + nimState.getNodeVal(node[1]) == "": # typedef struct X; # # (type_definition @@ -574,6 +607,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) + decho("addType(): case 2") nimState.addTypeObject(node[0]) else: let @@ -596,6 +630,7 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) + decho("addType(): case 3") nimState.addTypeTyped(node) elif not fdecl.isNil: # typedef X (*Y)(a1, a2, a3); @@ -625,6 +660,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) + decho("addType(): case 4") nimState.addTypeProc(node) elif not adecl.isNil: # typedef struct X Y[a][..]; @@ -646,6 +682,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) + decho("addType(): case 5") nimState.addTypeArray(node) else: if node.firstChildInTree("field_declaration_list").isNil: @@ -673,6 +710,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # First add struct as object + decho("addType(): case 6") nimState.addTypeObject(node[0]) if node.len > 1 and nimState.getNodeVal(node[1]) != "": @@ -684,6 +722,7 @@ proc addType(nimState: NimState, node: TSNode) = # typedef struct { .. } Y, *Z; # Get any name that isn't a pointer + decho("addType(): case 7") let name = block: var @@ -692,22 +731,21 @@ proc addType(nimState: NimState, node: TSNode) = if node[i].getName() == "type_identifier": name = nimState.getNodeVal(node[i].getAtom()) - if name.len == 0: - name = nimState.getUniqueIdentifier("STRUCT") - name # Now add struct as object with specified name nimState.addTypeObject(node[0], override = name) - if node.len > 1 and nimState.getNodeVal(node[1]) != "": + if name.len != 0: # Add any additional names except duplicate nimState.addTypeTyped(node, toverride = name, duplicate = name) proc addEnum(nimState: NimState, node: TSNode) = + decho("addEnum()") nimState.printDebug(node) proc addProc(nimState: NimState, node: TSNode) = + decho("addProc()") nimState.printDebug(node) proc processNode(nimState: NimState, node: TSNode): bool = diff --git a/nimterop/getters.nim b/nimterop/getters.nim index c25ce3c..84d7d83 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -313,7 +313,10 @@ proc anyChildInTree*(node: TSNode, ntype: string): TSNode = ccnode = cnode[i].anyChildInTree(ntype) if not ccnode.isNil(): return ccnode - cnode = cnode.tsNodeNextNamedSibling() + if cnode != node: + cnode = cnode.tsNodeNextNamedSibling() + else: + break proc mostNestedChildInTree*(node: TSNode): TSNode = # Search for the most nested child of node's type in tree @@ -444,6 +447,10 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = if offset.len == 0: echo "" +template decho*(str: untyped): untyped = + if nimState.gState.debug: + echo str.getCommented() + proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: echo ("Input => " & nimState.getNodeVal(node)).getCommented() diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d29ad6c..e123dd7 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,10 +1,11 @@ import sequtils, sets, tables -import compiler/[ast, idents, options] +import regex import "."/plugin when not declared(CIMPORT): + import compiler/[ast, idents, options] import "."/treesitter/api const @@ -50,9 +51,10 @@ type commentStr*, debugStr*, skipStr*: string # Nim compiler objects - constSection*, enumSection*, procSection*, typeSection*: PNode - identCache*: IdentCache - config*: ConfigRef + when not declared(CIMPORT): + constSection*, enumSection*, procSection*, typeSection*: PNode + identCache*: IdentCache + config*: ConfigRef gState*: State diff --git a/tests/include/test3.h b/tests/include/test3.h new file mode 100644 index 0000000..d1cf8aa --- /dev/null +++ b/tests/include/test3.h @@ -0,0 +1,42 @@ + +#define A 1 +#define B 1.0 +#define C 0x10 +#define D "hello" +#define E 'c' + +struct A0; +struct A1 {}; +typedef struct A2; +typedef struct A3 {}; +typedef struct A4 A4, *A4p; +typedef int A5; +typedef int *A6; +typedef A0 **A7; +typedef void *A8; + +typedef char *A9[3]; +typedef char *A10[3][6]; +typedef char *(*A11)[3]; + +typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int A13(int, int); + +struct A14 { char a1; }; +struct A15 { char *a1; int *a2[1]; }; + +typedef struct A16 { char f1; }; +typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p; +typedef struct { char *a1; int *a2[1]; } A19, *A19p; + +typedef struct A20 { char a1; } A20, A21, *A21p; + +//Expression +//typedef struct A21 { int **f1; int abc[123+132]; } A21; + +//Unions +//union UNION1 {int f1; }; +//typedef union UNION2 { int **f1; int abc[123+132]; } UNION2; + +// Anonymous +//typedef struct { char a1; }; diff --git a/tests/tnimterop2.nim b/tests/tnimterop2.nim new file mode 100644 index 0000000..1e9c21d --- /dev/null +++ b/tests/tnimterop2.nim @@ -0,0 +1,71 @@ +import tables + +import nimterop/[cimport] + +static: + cDebug() + +cImport("include/test3.h", flags="-d") + +proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = + var + obj: t + count = 0 + for name, value in obj.fieldPairs(): + count += 1 + assert name in fields, $t & "." & name & " invalid" + assert $fields[name] == $typeof(value), + "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value) + assert count == fields.len, "Failed for " & $t + +assert A == 1 +assert B == 1.0 +assert C == 0x10 +assert D == "hello" +assert E == 'c' + +assert A0 is object +testFields(A0) +assert A1 is object +testFields(A1) +assert A2 is object +testFields(A2) +assert A3 is object +testFields(A3) +assert A4 is object +testFields(A4) +assert A4p is ptr A4 +assert A5 is cint +assert A6 is ptr cint +assert A7 is ptr ptr A0 +assert A8 is pointer + +assert A9 is array[3, cstring] +assert A10 is array[3, array[6, cstring]] +assert A11 is ptr array[3, cstring] + +assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint +assert A13 is proc(a1: cint, a2: cint): cint + +assert A14 is object +testFields(A14, {"a1": "cchar"}.toTable()) + +assert A15 is object +testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) + +assert A16 is object +testFields(A16, {"f1": "cchar"}.toTable()) + +assert A17 is object +testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A18 is A17 +assert A18p is ptr A17 + +assert A19 is object +testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A19p is ptr A19 + +assert A20 is object +testFields(A20, {"a1": "cchar"}.toTable()) +assert A21 is A20 +assert A21p is ptr A20 From b1a56279ffc1870760520f19b6f4f9e17971fbc5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 9 Jan 2020 17:50:09 -0600 Subject: [PATCH 09/22] Fix CI, drop 0.19.6 support --- .travis.yml | 2 -- appveyor.yml | 1 - nimterop/all.nim | 2 +- nimterop/ast2.nim | 4 ++-- nimterop/build.nim | 8 +++----- nimterop/cimport.nim | 2 +- nimterop/docs.nim | 30 +++++++++++++----------------- nimterop/getters.nim | 10 +++++----- nimterop/toast.nim | 2 +- nimterop/types.nim | 27 ++++----------------------- 10 files changed, 30 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index 86e5fbc..359d07b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,14 +10,12 @@ addons: language: c env: - - BRANCH=0.19.6 - BRANCH=0.20.2 - BRANCH=1.0.4 - BRANCH=devel cache: directories: - - "$HOME/.choosenim/toolchains/nim-0.19.6" - "$HOME/.choosenim/toolchains/nim-0.20.2" - "$HOME/.choosenim/toolchains/nim-1.0.4" diff --git a/appveyor.yml b/appveyor.yml index 7678cf1..5651948 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,7 +9,6 @@ matrix: environment: matrix: - - NIM_VERSION: 0.19.6 - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.4 diff --git a/nimterop/all.nim b/nimterop/all.nim index c024d01..dc22967 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -4,4 +4,4 @@ Module that should import everything so that `nim doc --project nimtero/all` run # TODO: make sure it does import everything. -import "."/[cimport, build, types, plugin, compat] +import "."/[cimport, build, types, plugin] diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6c62b05..810beea 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,7 +4,7 @@ import compiler/[ast, astalgo, idents, options, renderer] import "."/treesitter/api -import "."/[compat, globals, getters] +import "."/[globals, getters] proc addConst(nimState: NimState, node: TSNode) = # #define X Y @@ -137,7 +137,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = proc getTypeArray(nimState: NimState, node: TSNode): PNode proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode -proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64, exported = false): PNode = +proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field # # For proc, param should not be exported diff --git a/nimterop/build.nim b/nimterop/build.nim index 6ac21e9..3eec406 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep -import "."/[compat] - proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: @@ -648,7 +646,7 @@ proc getGccPaths*(mode = "c"): seq[string] = break if inc: var - path = line.strip().myNormalizedPath() + path = line.strip().normalizedPath() if path notin result: result.add path @@ -667,13 +665,13 @@ proc getGccLibPaths*(mode = "c"): seq[string] = if "LIBRARY_PATH=" in line: for path in line[13 .. ^1].split(PathSep): var - path = path.strip().myNormalizedPath() + path = path.strip().normalizedPath() if path notin result: result.add path break elif '\t' in line: var - path = line.strip().myNormalizedPath() + path = line.strip().normalizedPath() if path notin result: result.add path diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 4c63c31..84f7b4f 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1 include "."/globals -import "."/[build, compat, paths, types] +import "."/[build, paths, types] export types proc interpPath(dir: string): string= diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 17089ec..7ea797a 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -1,22 +1,18 @@ import macros, strformat -when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): - from os import parentDir, getCurrentCompilerExe, DirSep - proc getNimRootDir(): string = - #[ - hack, but works - alternatively (but more complex), use (from a nim file, not nims otherwise - you get Error: ambiguous call; both system.fileExists): - import "$nim/testament/lib/stdtest/specialpaths.nim" - nimRootDir - ]# - fmt"{currentSourcePath}".parentDir.parentDir.parentDir -else: - proc getCurrentCompilerExe*(): string = - "nim" +from os import parentDir, getCurrentCompilerExe, DirSep +proc getNimRootDir(): string = + #[ + hack, but works + alternatively (but more complex), use (from a nim file, not nims otherwise + you get Error: ambiguous call; both system.fileExists): + import "$nim/testament/lib/stdtest/specialpaths.nim" + nimRootDir + ]# + fmt"{currentSourcePath}".parentDir.parentDir.parentDir - const - DirSep = when defined(windows): '\\' else: '/' +const + DirSep = when defined(windows): '\\' else: '/' proc execAction(cmd: string): string = var @@ -80,4 +76,4 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath for i in 0 .. paramCount(): if paramStr(i) == "--publish": echo execAction(&"cd {path} && ghp-import --no-jekyll -fp {path}") - break \ No newline at end of file + break diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 84d7d83..2469d1c 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -4,7 +4,7 @@ import regex import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] -import "."/[build, compat, globals, plugin, treesitter/api] +import "."/[build, globals, plugin, treesitter/api] const gReserved = """ addr and as asm @@ -236,11 +236,11 @@ proc getLit*(str: string): PNode = proc isNil*(node: TSNode): bool = node.tsNodeIsNull() -proc len*(node: TSNode): uint = +proc len*(node: TSNode): int = if not node.isNil: - result = node.tsNodeNamedChildCount().uint + result = node.tsNodeNamedChildCount().int -proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = +proc `[]`*(node: TSNode, i: SomeInteger): TSNode = if i < node.len(): result = node.tsNodeNamedChild(i.uint32) @@ -652,6 +652,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandSymlink().absolutePath(path.parentDir()).myNormalizedPath() + result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath() except: result = path diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 26f9cdc..f775738 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,7 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast2, compat, globals, getters] +import "."/[ast2, globals, getters] proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/types.nim b/nimterop/types.nim index a57252f..89f779d 100644 --- a/nimterop/types.nim +++ b/nimterop/types.nim @@ -1,26 +1,7 @@ -# see https://github.com/nimterop/nimterop/issues/79 - -when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): - # clean this up once upgraded; adapted from std/time_t - when defined(nimdoc): - type - impl = distinct int64 - Time = impl - elif defined(windows): - when defined(i386) and defined(gcc): - type Time {.importc: "time_t", header: "".} = distinct int32 - else: - type Time {.importc: "time_t", header: "".} = distinct int64 - elif defined(posix): - import posix - type - time_t* = Time - time64_t* = Time -else: - import std/time_t as time_t_temp - type - time_t* = time_t_temp.Time - time64_t* = time_t_temp.Time +import std/time_t as time_t_temp +type + time_t* = time_t_temp.Time + time64_t* = time_t_temp.Time when defined(cpp): # http://www.cplusplus.com/reference/cwchar/wchar_t/ From 8eb586fd8064486612e19fb18e02b714ad803067 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jan 2020 16:48:45 -0600 Subject: [PATCH 10/22] Fix bug --- nimterop/getters.nim | 4 ++-- nimterop/globals.nim | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 94bfbb3..f123336 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -449,8 +449,8 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: - necho ("Input => " & nimState.getNodeVal(node)).getCommented() - necho nimState.gState.printLisp(node).getCommented() + necho ("Input => " & nimState.getNodeVal(node)).getCommented() & "\n" & + nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index cc7d499..9db0013 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -99,5 +99,4 @@ when not declared(CIMPORT): template decho*(str: untyped): untyped = if nimState.gState.debug: - let gState = nimState.gState necho str.getCommented() From 0ae87e82f54257d514faeaf2ad201bb263fcce5f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 29 Dec 2019 04:46:19 -0600 Subject: [PATCH 11/22] New Nim AST based backend - #defines working --- nimterop/ast2.nim | 133 ++++++++++++++++++++ nimterop/getters.nim | 292 ++++++++++++++++++++++--------------------- nimterop/globals.nim | 29 ++--- nimterop/nim.cfg | 1 + nimterop/toast.nim | 20 +-- 5 files changed, 298 insertions(+), 177 deletions(-) create mode 100644 nimterop/ast2.nim create mode 100644 nimterop/nim.cfg diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim new file mode 100644 index 0000000..dc62d4f --- /dev/null +++ b/nimterop/ast2.nim @@ -0,0 +1,133 @@ +import macros, os, strutils, tables, times + +import compiler/[ast, idents, options, renderer] + +import "."/treesitter/api + +import "."/[compat, globals, getters] + +proc addConst(nimState: NimState, node: TSNode) = + echo "Const: " & nimState.getNodeVal(node) + if node.len() >= 2: + let + constDef = newNode(nkConstDef) + (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) + ident = nimState.getIdent(name, info) + val = nimState.getNodeVal(node[1]).getLit() + if val.kind != nkNilLit: + constDef.add ident + constDef.add newNode(nkEmpty) + constDef.add val + + nimState.constSection.add constDef + +proc addType(nimState: NimState, node: TSNode) = + echo "Type: " & nimState.getNodeVal(node) + +proc addEnum(nimState: NimState, node: TSNode) = + echo "Enum: " & nimState.getNodeVal(node) + +proc addProc(nimState: NimState, node: TSNode) = + echo "Proc: " & nimState.getNodeVal(node) + +proc processNode(nimState: NimState, node: TSNode): bool = + result = true + + case node.getName() + of "preproc_def": + nimState.addConst(node) + of "type_definition": + if node.inTree("struct_specifier"): + nimState.addType(node) + elif node.inTree("enum_specifier"): + nimState.addEnum(node) + else: + # Unknown type + result = false + of "struct_specifier": + nimState.addType(node) + of "enum_specifier": + nimState.addEnum(node) + of "declaration": + nimState.addProc(node) + else: + # Unknown + result = false + +proc searchTree(nimState: NimState, root: TSNode) = + # Search AST generated by tree-sitter for recognized elements + var + node = root + nextnode: TSNode + depth = 0 + processed = false + + while true: + if not node.tsNodeIsNull() and depth > -1: + processed = nimState.processNode(node) + else: + break + + if not processed and node.len() != 0: + nextnode = node[0] + depth += 1 + else: + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.tsNodeIsNull(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if node == root: + break + if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc printNimHeader*() = + echo """# Generated at $1 +# Command line: +# $2 $3 + +{.hint[ConvFromXtoItselfNotNeeded]: off.} + +import nimterop/types +""" % [$now(), getAppFilename(), commandLineParams().join(" ")] + +proc printNim*(gState: State, fullpath: string, root: TSNode) = + var + nimState = new(NimState) + fp = fullpath.replace("\\", "/") + + nimState.identifiers = newTable[string, string]() + + nimState.gState = gState + nimState.currentHeader = getCurrentHeader(fullpath) + nimState.sourceFile = fullpath + + # Nim compiler objects + nimState.identCache = newIdentCache() + nimState.config = newConfigRef() + + nimState.constSection = newNode(nkConstSection) + nimState.enumSection = newNode(nkStmtList) + nimState.procSection = newNode(nkStmtList) + nimState.typeSection = newNode(nkTypeSection) + + nimState.searchTree(root) + + var + tree = newNode(nkStmtList) + tree.add nimState.enumSection + tree.add nimState.constSection + tree.add nimState.typeSection + tree.add nimState.procSection + + echo tree.renderTree() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 885ce2f..52a77a6 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,6 +2,8 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex +import compiler/[ast, idents, lineinfos, msgs, pathutils] + import "."/[build, compat, globals, plugin, treesitter/api] const gReserved = """ @@ -26,6 +28,8 @@ when while xor yield""".split(Whitespace).toHashSet() +# Types related + const gTypeMap* = { # char "char": "cchar", @@ -91,6 +95,21 @@ proc getType*(str: string): string = if gTypeMap.hasKey(result): result = gTypeMap[result] +proc getPtrType*(str: string): string = + result = case str: + of "ptr cchar": + "cstring" + of "ptr ptr cchar": + "ptr cstring" + of "ptr object": + "pointer" + of "ptr ptr object": + "ptr pointer" + else: + str + +# Identifier related + proc checkIdentifier(name, kind, parent, origName: string) = let parentStr = if parent.nBl: parent & ":" else: "" @@ -165,6 +184,8 @@ proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool nimState.identifiers[nimName] = name result = true +# Overrides related + proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string = doAssert name.nBl, "Blank identifier error" @@ -189,34 +210,96 @@ proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string = for i in nimState.gState.onSymbolOverrideFinal(typ): result &= "\n" & nimState.getOverride(i, kind) -proc getPtrType*(str: string): string = - result = case str: - of "ptr cchar": - "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": - "pointer" - of "ptr ptr object": - "ptr pointer" - else: - str - -proc getLit*(str: string): string = +proc getLit*(str: string): PNode = # Used to convert #define literals into const let str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal - str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal - str.contains(re"^'[[:ascii:]]'$") or # char - str.contains(re"""^"[[:ascii:]]+"$"""): # char * - return str + if str.contains(re"^[\-]?[\d]+$"): # decimal + result = newIntNode(nkIntLit, parseInt(str)) + + elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float + result = newFloatNode(nkFloatLit, parseFloat(str)) + + elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal + result = newIntNode(nkIntLit, parseHexInt(str)) + + elif str.contains(re"^'[[:ascii:]]'$"): # char + result = newNode(nkCharLit) + result.intVal = str[1].int64 + + elif str.contains(re"""^"[[:ascii:]]+"$"""): # char * + result = newStrNode(nkStrLit, str[1 .. ^2]) + + else: + result = newNode(nkNilLit) + +# TSNode shortcuts + +proc len*(node: TSNode): uint = + if not node.tsNodeIsNull: + result = node.tsNodeNamedChildCount().uint + +proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = + if i < node.len(): + result = node.tsNodeNamedChild(i.uint32) + +proc getName*(node: TSNode): string {.inline.} = + if not node.tsNodeIsNull: + return $node.tsNodeType() proc getNodeVal*(nimState: NimState, node: TSNode): string = - return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + if not node.tsNodeIsNull: + return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + +proc getAtom*(node: TSNode): TSNode = + if not node.tsNodeIsNull: + # Get child node which is topmost atom + if node.getName() in gAtoms: + return node + elif node.len() != 0: + return node[0].getAtom() + +proc getPtrCount*(node: TSNode): string = + if not node.tsNodeIsNull: + # Get number of ptr nodes in tree + var + cnode = node + while "pointer_declarator" in cnode.getName(): + result &= "ptr " + if cnode.len() != 0: + cnode = cnode[0] + else: + break + +proc getDeclarator*(node: TSNode): TSNode = + if not node.tsNodeIsNull: + # Return if child is a function or array declarator + if node.getName() in ["function_declarator", "array_declarator"]: + return node + elif node.len() != 0: + return node[0].getDeclarator() + +proc inTree*(node: TSNode, ntype: string): bool = + # Search for node type in tree - first children + result = false + var + cnode = node + while not cnode.tsNodeIsNull: + if cnode.getName() == ntype: + return true + cnode = cnode[0] + +proc inChildren*(node: TSNode, ntype: string): bool = + # Search for node type in immediate children + result = false + for i in 0 ..< node.len(): + if (node[i]).getName() == ntype: + result = true + break proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node result.line = 1 result.col = 1 for i in 0 .. node.tsNodeStartByte().int-1: @@ -225,6 +308,54 @@ proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = result.line += 1 result.col += 1 +proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = + for i in 0 ..< node.len(): + if node.getName() != "comment": + result += 1 + +proc getPxName*(node: TSNode, offset: int): string = + # Get the xth (grand)parent of the node + var + np = node + count = 0 + + while not np.tsNodeIsNull() and count < offset: + np = np.tsNodeParent() + count += 1 + + if count == offset and not np.tsNodeIsNull(): + return np.getName() + +# Compiler shortcuts + +proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = + # Get Nim equivalent line:col info from node + let + (line, col) = nimState.gState.getLineCol(node) + + result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col) + +proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode = + # Get ident PNode for name + info + let + exp = getIdent(nimState.identCache, "*") + ident = getIdent(nimState.identCache, name) + + if exported: + result = newNode(nkPostfix) + result.add newIdentNode(exp, info) + result.add newIdentNode(ident, info) + else: + result = newIdentNode(ident, info) + +proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""): + tuple[name: string, info: TLineInfo] = + # Shortcut to get identifier name and info (node value and line:col) + let + name = nimState.getNodeVal(node) + result.name = nimState.getIdentifier(name, kind, parent) + result.info = nimState.getLineInfo(node) + proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) @@ -291,91 +422,6 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string = replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1"). removeStatic() -converter toString*(kind: Kind): string = - return case kind: - of exactlyOne: - "" - of oneOrMore: - "+" - of zeroOrMore: - "*" - of zeroOrOne: - "?" - of orWithNext: - "!" - -converter toKind*(kind: string): Kind = - return case kind: - of "+": - oneOrMore - of "*": - zeroOrMore - of "?": - zeroOrOne - of "!": - orWithNext - else: - exactlyOne - -proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool] = - if name[0] == '^': - result.recursive = true - result.name = name[1 .. ^1] - else: - result.name = name - result.kind = $name[^1] - - if result.kind != exactlyOne: - result.name = result.name[0 .. ^2] - -proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - if $node.tsNodeType() != "comment": - result += 1 - -proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = - if node.tsNodeNamedChildCount() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - let - name = $node.tsNodeNamedChild(i).tsNodeType() - - if name != "comment": - result.add(name) - -proc getRegexForAstChildren*(ast: ref Ast): string = - result = "^" - for i in 0 .. ast.children.len-1: - let - kind: string = ast.children[i].kind - begin = if result[^1] == '|': "" else: "(?:" - case kind: - of "!": - result &= &"{begin}{ast.children[i].name}|" - else: - result &= &"{begin}{ast.children[i].name}){kind}" - result &= "$" - -proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = - for i in 0 .. ast.children.len-1: - if name in ast.children[i].name.split("|"): - return ast.children[i] - - if ast.children.len == 1 and ast.children[0].name == ".": - return ast.children[0] - -proc getPxName*(node: TSNode, offset: int): string = - var - np = node - count = 0 - - while not np.tsNodeIsNull() and count < offset: - np = np.tsNodeParent() - count += 1 - - if count == offset and not np.tsNodeIsNull(): - return $np.tsNodeType() - proc getNimExpression*(nimState: NimState, expr: string): string = var clean = expr.multiReplace([("\n", " "), ("\r", "")]) @@ -435,42 +481,6 @@ proc getSplitComma*(joined: seq[string]): seq[string] = for i in joined: result = result.concat(i.split(",")) -proc getHeader*(nimState: NimState): string = - result = - if nimState.gState.dynlib.Bl: - &", header: {nimState.currentHeader}" - else: - "" - -proc getDynlib*(nimState: NimState): string = - result = - if nimState.gState.dynlib.nBl: - &", dynlib: {nimState.gState.dynlib}" - else: - "" - -proc getImportC*(nimState: NimState, origName, nimName: string): string = - if nimName != origName: - result = &"importc: \"{origName}\"{nimState.getHeader()}" - else: - result = nimState.impShort - -proc getPragma*(nimState: NimState, pragmas: varargs[string]): string = - result = "" - for pragma in pragmas.items(): - if pragma.nBl: - result &= pragma & ", " - if result.nBl: - result = " {." & result[0 .. ^3] & ".}" - - result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C") - - let - dy = nimState.getDynlib() - - if ", cdecl" in result and dy.nBl: - result = result.replace(".}", dy & ".}") - proc getComments*(nimState: NimState, strip = false): string = if not nimState.gState.nocomments and nimState.commentStr.nBl: result = "\n" & nimState.commentStr diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 52343de..1921b1b 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,6 +1,6 @@ import sequtils, sets, tables -import regex +import compiler/[ast, idents, options] import "."/plugin @@ -34,24 +34,6 @@ const ].concat(toSeq(gExpressions.items)) type - Kind = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - when not declared(CIMPORT): - tonim*: proc (ast: ref Ast, node: TSNode, nimState: NimState) - regex*: Regex - - AstTable {.used.} = TableRef[string, seq[ref Ast]] - State = ref object compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string] @@ -67,7 +49,12 @@ type NimState {.used.} = ref object identifiers*: TableRef[string, string] - commentStr*, constStr*, debugStr*, enumStr*, procStr*, skipStr*, typeStr*: string + commentStr*, debugStr*, skipStr*: string + + # Nim compiler objects + constSection*, enumSection*, procSection*, typeSection*: PNode + identCache*: IdentCache + config*: ConfigRef gState*: State @@ -94,7 +81,7 @@ type CompileMode = enum const modeDefault {.used.} = $cpp # TODO: USE this everywhere relevant when not declared(CIMPORT): - export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, + export gAtoms, gExpressions, gEnumVals, State, NimState, nBl, Bl, CompileMode, modeDefault # Redirect output to file when required diff --git a/nimterop/nim.cfg b/nimterop/nim.cfg new file mode 100644 index 0000000..913df8c --- /dev/null +++ b/nimterop/nim.cfg @@ -0,0 +1 @@ +--path:"$nim" diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 76604f1..1ad10a2 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,7 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast, compat, globals, getters, grammar] +import "."/[ast2, compat, globals, getters] proc printLisp(gState: State, root: TSNode) = var @@ -53,7 +53,7 @@ proc printLisp(gState: State, root: TSNode) = if node == root: break -proc process(gState: State, path: string, astTable: AstTable) = +proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" var @@ -93,7 +93,7 @@ proc process(gState: State, path: string, astTable: AstTable) = if gState.past: gState.printLisp(root) elif gState.pnim: - gState.printNim(path, root, astTable) + gState.printNim(path, root) elif gState.preprocess: gecho gState.code @@ -109,7 +109,6 @@ proc main( nocomments = false, output = "", past = false, - pgrammar = false, pluginSourcePath: string = "", pnim = false, prefix: seq[string] = @[], @@ -167,19 +166,12 @@ proc main( doAssert gState.outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}" - # Process grammar into AST - let - astTable = parseGrammar() - - if pgrammar: - # Print AST of grammar - gState.printGrammar(astTable) - elif source.nBl: + if source.nBl: # Print source after preprocess or Nim output if gState.pnim: gState.printNimHeader() for src in source: - gState.process(src.expandSymlinkAbs(), astTable) + gState.process(src.expandSymlinkAbs()) # Close outputFile if outputFile.len != 0: @@ -241,7 +233,6 @@ when isMainModule: "nocomments": "exclude top-level comments from output", "output": "file to output content - default stdout", "past": "print AST output", - "pgrammar": "print grammar", "pluginSourcePath": "Nim file to build and load as a plugin", "pnim": "print Nim output", "preprocess": "run preprocessor on header", @@ -260,7 +251,6 @@ when isMainModule: "nocomments": 'c', "output": 'o', "past": 'a', - "pgrammar": 'g', "pnim": 'n', "prefix": 'E', "preprocess": 'p', From 248761da11e03d4cf78739d7af9073e6179d0b8b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 1 Jan 2020 16:37:33 -0600 Subject: [PATCH 12/22] Print debug, more comments --- config.nims | 5 ++- nimterop/ast2.nim | 75 ++++++++++++++++++++++++++++++++--- nimterop/getters.nim | 93 +++++++++++++++++++++++++++++++++++++++++++- nimterop/nim.cfg | 1 - nimterop/toast.nim | 51 +----------------------- 5 files changed, 166 insertions(+), 59 deletions(-) delete mode 100644 nimterop/nim.cfg diff --git a/config.nims b/config.nims index 54a9a06..d312c07 100644 --- a/config.nims +++ b/config.nims @@ -9,4 +9,7 @@ when defined(Windows): switch("gc", "markAndSweep") # Retain stackTrace for clear errors -switch("stackTrace", "on") \ No newline at end of file +switch("stackTrace", "on") + +# Path to compiler +switch("path", "$nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dc62d4f..8790e6f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,34 +1,97 @@ import macros, os, strutils, tables, times -import compiler/[ast, idents, options, renderer] +import compiler/[ast, astalgo, idents, options, renderer] import "."/treesitter/api import "."/[compat, globals, getters] proc addConst(nimState: NimState, node: TSNode) = - echo "Const: " & nimState.getNodeVal(node) - if node.len() >= 2: + # (preproc_def + # (identifier) + # (preproc_arg) + # ) + nimState.printDebug(node) + + if node[0].getName() == "identifier" and + node[1].getName() == "preproc_arg": let constDef = newNode(nkConstDef) + + # node[0] = identifier = const name (name, info) = nimState.getNameInfo(node.getAtom(), nskConst) + # TODO - check blank and override ident = nimState.getIdent(name, info) + + # node[1] = preproc_arg = value val = nimState.getNodeVal(node[1]).getLit() + + # If supported literal if val.kind != nkNilLit: + # nnkConstDef( + # nnkPostfix( + # nnkIdent("*"), + # nnkIdent(name) + # ), + # nnkEmpty(), + # nnkXLit(val) + # ) constDef.add ident constDef.add newNode(nkEmpty) constDef.add val + # nnkConstSection.add nimState.constSection.add constDef + nimState.printDebug(constDef) + proc addType(nimState: NimState, node: TSNode) = - echo "Type: " & nimState.getNodeVal(node) + # CASE1: + # + # typedef struct X Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_identifier) + # ) + # + # CASE 2 + # + # typedef struct X *Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator + # (type_identifier) + # ) + # ) + # + nimState.printDebug(node) + + let + typeDef = newNode(nkTypeDef) + + fdlist = node.inTree("field_declaration_list") + + identNode = if fdlist: node[1] else: node + + (name, info) = nimState.getNameInfo(identNode.getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info) + + typeNode = + if fdlist: node[0] else: node[1] + proc addEnum(nimState: NimState, node: TSNode) = - echo "Enum: " & nimState.getNodeVal(node) + nimState.printDebug(node) proc addProc(nimState: NimState, node: TSNode) = - echo "Proc: " & nimState.getNodeVal(node) + nimState.printDebug(node) proc processNode(nimState: NimState, node: TSNode): bool = result = true diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 52a77a6..5e53f94 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import compiler/[ast, idents, lineinfos, msgs, pathutils] +import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] import "."/[build, compat, globals, plugin, treesitter/api] @@ -221,6 +221,7 @@ proc getLit*(str: string): PNode = elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"): # float result = newFloatNode(nkFloatLit, parseFloat(str)) + # TODO - hex becomes int on render elif str.contains(re"^0x[\da-fA-F]+$"): # hexadecimal result = newIntNode(nkIntLit, parseHexInt(str)) @@ -326,6 +327,96 @@ proc getPxName*(node: TSNode, offset: int): string = if count == offset and not np.tsNodeIsNull(): return np.getName() +proc printLisp*(gState: State, root: TSNode): string = + var + node = root + nextnode: TSNode + depth = 0 + + while true: + if not node.tsNodeIsNull() and depth > -1: + if gState.pretty: + result &= spaces(depth) + let + (line, col) = gState.getLineCol(node) + result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + else: + break + + if node.tsNodeNamedChildCount() != 0: + if gState.pretty: + result &= "\n" + nextnode = node.tsNodeNamedChild(0) + depth += 1 + else: + if gState.pretty: + result &= ")\n" + else: + result &= ")" + nextnode = node.tsNodeNextNamedSibling() + + if nextnode.tsNodeIsNull(): + while true: + node = node.tsNodeParent() + depth -= 1 + if depth == -1: + break + if gState.pretty: + result &= spaces(depth) & ")\n" + else: + result &= ")" + if node == root: + break + if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + node = node.tsNodeNextNamedSibling() + break + else: + node = nextnode + + if node == root: + break + +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +proc printTree*(nimState: NimState, pnode: PNode, offset = "") = + if nimState.gState.debug and pnode.kind != nkNone: + stdout.write "\n# " & offset & $pnode.kind & "(" + case pnode.kind + of nkCharLit: + stdout.write "'" & pnode.intVal.char & "')" + of nkIntLit..nkUInt64Lit: + stdout.write $pnode.intVal & ")" + of nkFloatLit..nkFloat128Lit: + stdout.write $pnode.floatVal & ")" + of nkStrLit..nkTripleStrLit: + stdout.write "\"" & pnode.strVal & "\")" + of nkSym: + stdout.write $pnode.sym & ")" + of nkIdent: + stdout.write "\"" & $pnode.ident.s & "\")" + else: + if pnode.sons.len != 0: + for i in 0 ..< pnode.sons.len: + nimState.printTree(pnode.sons[i], offset & " ") + if i != pnode.sons.len - 1: + stdout.write "," + stdout.write "\n# " & offset & ")" + else: + stdout.write ")" + if offset.len == 0: + echo "" + +proc printDebug*(nimState: NimState, node: TSNode) = + if nimState.gState.debug: + echo nimState.getNodeVal(node).getCommented() + echo nimState.gState.printLisp(node).getCommented() + +proc printDebug*(nimState: NimState, pnode: PNode) = + if nimState.gState.debug: + echo ($pnode).getCommented() + nimState.printTree(pnode) + # Compiler shortcuts proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo = diff --git a/nimterop/nim.cfg b/nimterop/nim.cfg deleted file mode 100644 index 913df8c..0000000 --- a/nimterop/nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---path:"$nim" diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 1ad10a2..db68ce4 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,55 +4,6 @@ import "."/treesitter/[api, c, cpp] import "."/[ast2, compat, globals, getters] -proc printLisp(gState: State, root: TSNode) = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.tsNodeIsNull() and depth > -1: - if gState.pretty: - stdout.write spaces(depth) - let - (line, col) = gState.getLineCol(node) - stdout.write &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" - else: - break - - if node.tsNodeNamedChildCount() != 0: - if gState.pretty: - gecho "" - nextnode = node.tsNodeNamedChild(0) - depth += 1 - else: - if gState.pretty: - gecho ")" - else: - stdout.write ")" - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.tsNodeIsNull(): - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - if gState.pretty: - gecho spaces(depth) & ")" - else: - stdout.write ")" - if node == root: - break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break - proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -91,7 +42,7 @@ proc process(gState: State, path: string) = tree.tsTreeDelete() if gState.past: - gState.printLisp(root) + echo gState.printLisp(root) elif gState.pnim: gState.printNim(path, root) elif gState.preprocess: From 68638c8bb6da233d8dd665c580a13c6a9e85b4b0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jan 2020 15:59:14 -0600 Subject: [PATCH 13/22] More types, isNil, data in lisp --- nimterop/ast2.nim | 245 ++++++++++++++++++++++++++++++++++++------- nimterop/getters.nim | 65 ++++++++---- 2 files changed, 251 insertions(+), 59 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 8790e6f..6f8f543 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -7,6 +7,8 @@ import "."/treesitter/api import "."/[compat, globals, getters] proc addConst(nimState: NimState, node: TSNode) = + # #define X Y + # # (preproc_def # (identifier) # (preproc_arg) @@ -28,35 +30,130 @@ proc addConst(nimState: NimState, node: TSNode) = # If supported literal if val.kind != nkNilLit: - # nnkConstDef( - # nnkPostfix( - # nnkIdent("*"), - # nnkIdent(name) - # ), - # nnkEmpty(), - # nnkXLit(val) + # const X* = Y + # + # nkConstDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkXLit(Y) # ) constDef.add ident constDef.add newNode(nkEmpty) constDef.add val - # nnkConstSection.add + # nkConstSection.add nimState.constSection.add constDef nimState.printDebug(constDef) +proc newTypeIdent(nimState: NimState, node: TSNode): PNode = + # Create nkTypeDef PNode with first ident + let + (name, info) = nimState.getNameInfo(node.getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info) + + result = newNode(nkTypeDef) + result.add ident + result.add newNode(nkEmpty) + +proc newPtrTree(count: int): tuple[parent, child: PNode] = + # Create nkPtrTy tree depending on count + # + # nkPtrTy( + # nkPtrTy( + # .. + # ) + # ) + if count > 0: + result.parent = newNode(nkPtrTy) + result.child = result.parent + for i in 1 ..< count: + let + child = newNode(nkPtrTy) + result.child.add child + result.child = child + +proc addType0(nimState: NimState, node: TSNode) = + let + typeDef = nimState.newTypeIdent(node) + + # type X* = object + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkEmpty() + # ) + # ) + typeDef.add(block: + let + obj = newNode(nkObjectTy) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + + obj + ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addType1(nimState: NimState, node: TSNode) = + let + # node[1] = identifer = name + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = type name + (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) + # TODO - check blank and override + # TODO - use getPtrType() - ptr cchar to cstring + ident = nimState.getIdent(name, info, exported = false) + + # node[1] could have pointers + count = node[1].getPtrCount() + + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + + if count > 0: + let + (parent, child) = newPtrTree(count) + child.add ident + typeDef.add parent + else: + typeDef.add ident + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + proc addType(nimState: NimState, node: TSNode) = # CASE1: # - # typedef struct X Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (type_identifier) - # ) - # + # # CASE 2 # # typedef struct X *Y; @@ -72,20 +169,93 @@ proc addType(nimState: NimState, node: TSNode) = # nimState.printDebug(node) - let - typeDef = newNode(nkTypeDef) - - fdlist = node.inTree("field_declaration_list") - - identNode = if fdlist: node[1] else: node - - (name, info) = nimState.getNameInfo(identNode.getAtom(), nskType) - # TODO - check blank and override - ident = nimState.getIdent(name, info) - - typeNode = - if fdlist: node[0] else: node[1] - + if node.getName() == "struct_specifier": + if node.len == 1: + # struct X; + # + # (struct_specifier + # (type_identifier) + # ) + nimState.addType0(node) + elif node.len == 2: + if node[1].getName() == "field_declaration_list" and node[1].len == 0: + # struct X {}; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + nimState.addType0(node) + elif node.getName() == "type_definition": + if node.len == 2: + let + fdlist = node[0].anyChildInTree("field_declaration_list") + if nimState.getNodeVal(node[1]) == "": + if fdlist.isNil(): + # typedef struct X; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_definition = "") + # ) + nimState.addType0(node) + elif fdlist.len == 0: + # typedef struct X {}; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # (type_definition = "") + # ) + nimState.addType0(node) + else: + let + sspec = node[0].firstChildInTree("struct_specifier") + adecl = node[1].anyChildInTree("array_declarator") + fdecl = node[1].anyChildInTree("function_declarator") + if fdlist.isNil(): + if adecl.isNil and fdecl.isNil: + if not sspec.isNil: + # typedef struct X [*]Y; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addType1(node) + else: + # typedef struct int Y; + # + # (type_definition + # (type_identifier|primitive_type) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addType1(node) + elif not adecl.isNil: + # typedef X Y[a][..]; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator + # (array_declarator + # (type_identifier) + # (number_literal) + # ) + # ) + # ) + discard proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -100,13 +270,10 @@ proc processNode(nimState: NimState, node: TSNode): bool = of "preproc_def": nimState.addConst(node) of "type_definition": - if node.inTree("struct_specifier"): - nimState.addType(node) - elif node.inTree("enum_specifier"): + if not node.firstChildInTree("enum_specifier").isNil(): nimState.addEnum(node) else: - # Unknown type - result = false + nimState.addType(node) of "struct_specifier": nimState.addType(node) of "enum_specifier": @@ -126,7 +293,7 @@ proc searchTree(nimState: NimState, root: TSNode) = processed = false while true: - if not node.tsNodeIsNull() and depth > -1: + if not node.isNil() and depth > -1: processed = nimState.processNode(node) else: break @@ -137,7 +304,7 @@ proc searchTree(nimState: NimState, root: TSNode) = else: nextnode = node.tsNodeNextNamedSibling() - if nextnode.tsNodeIsNull(): + if nextnode.isNil(): while true: node = node.tsNodeParent() depth -= 1 @@ -145,7 +312,7 @@ proc searchTree(nimState: NimState, root: TSNode) = break if node == root: break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + if not node.tsNodeNextNamedSibling().isNil(): node = node.tsNodeNextNamedSibling() break else: diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 5e53f94..2269268 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -237,8 +237,11 @@ proc getLit*(str: string): PNode = # TSNode shortcuts +proc isNil*(node: TSNode): bool = + node.tsNodeIsNull() + proc len*(node: TSNode): uint = - if not node.tsNodeIsNull: + if not node.isNil: result = node.tsNodeNamedChildCount().uint proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = @@ -246,51 +249,67 @@ proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = result = node.tsNodeNamedChild(i.uint32) proc getName*(node: TSNode): string {.inline.} = - if not node.tsNodeIsNull: + if not node.isNil: return $node.tsNodeType() +proc getNodeVal*(gState: State, node: TSNode): string = + if not node.isNil: + return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + proc getNodeVal*(nimState: NimState, node: TSNode): string = - if not node.tsNodeIsNull: - return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + nimState.gState.getNodeVal(node) proc getAtom*(node: TSNode): TSNode = - if not node.tsNodeIsNull: + if not node.isNil: # Get child node which is topmost atom if node.getName() in gAtoms: return node elif node.len() != 0: return node[0].getAtom() -proc getPtrCount*(node: TSNode): string = - if not node.tsNodeIsNull: +proc getPtrCount*(node: TSNode): int = + if not node.isNil: # Get number of ptr nodes in tree var cnode = node while "pointer_declarator" in cnode.getName(): - result &= "ptr " + result += 1 if cnode.len() != 0: cnode = cnode[0] else: break proc getDeclarator*(node: TSNode): TSNode = - if not node.tsNodeIsNull: + if not node.isNil: # Return if child is a function or array declarator if node.getName() in ["function_declarator", "array_declarator"]: return node elif node.len() != 0: return node[0].getDeclarator() -proc inTree*(node: TSNode, ntype: string): bool = +proc firstChildInTree*(node: TSNode, ntype: string): TSNode = # Search for node type in tree - first children - result = false var cnode = node - while not cnode.tsNodeIsNull: + while not cnode.isNil: if cnode.getName() == ntype: - return true + return cnode cnode = cnode[0] +proc anyChildInTree*(node: TSNode, ntype: string): TSNode = + # Search for node type anywhere in tree - depth first + var + cnode = node + while not cnode.isNil: + if cnode.getName() == ntype: + return cnode + for i in 0 ..< cnode.len: + let + ccnode = cnode[i].anyChildInTree(ntype) + if not ccnode.isNil(): + return ccnode + cnode = cnode.tsNodeNextNamedSibling() + proc inChildren*(node: TSNode, ntype: string): bool = # Search for node type in immediate children result = false @@ -320,11 +339,11 @@ proc getPxName*(node: TSNode, offset: int): string = np = node count = 0 - while not np.tsNodeIsNull() and count < offset: + while not np.isNil() and count < offset: np = np.tsNodeParent() count += 1 - if count == offset and not np.tsNodeIsNull(): + if count == offset and not np.isNil(): return np.getName() proc printLisp*(gState: State, root: TSNode): string = @@ -334,12 +353,16 @@ proc printLisp*(gState: State, root: TSNode): string = depth = 0 while true: - if not node.tsNodeIsNull() and depth > -1: + if not node.isNil() and depth > -1: if gState.pretty: result &= spaces(depth) let (line, col) = gState.getLineCol(node) result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}" + let + val = gState.getNodeVal(node) + if "\n" notin val and " " notin val: + result &= &" \"{val}\"" else: break @@ -355,7 +378,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= ")" nextnode = node.tsNodeNextNamedSibling() - if nextnode.tsNodeIsNull(): + if nextnode.isNil(): while true: node = node.tsNodeParent() depth -= 1 @@ -367,7 +390,7 @@ proc printLisp*(gState: State, root: TSNode): string = result &= ")" if node == root: break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): + if not node.tsNodeNextNamedSibling().isNil(): node = node.tsNodeNextNamedSibling() break else: @@ -409,12 +432,12 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: - echo nimState.getNodeVal(node).getCommented() + echo ("Input => " & nimState.getNodeVal(node)).getCommented() echo nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: - echo ($pnode).getCommented() + echo ("Output => " & $pnode).getCommented() nimState.printTree(pnode) # Compiler shortcuts @@ -445,6 +468,8 @@ proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = " let name = nimState.getNodeVal(node) result.name = nimState.getIdentifier(name, kind, parent) + if kind == nskType: + result.name = result.name.getType() result.info = nimState.getLineInfo(node) proc getCurrentHeader*(fullpath: string): string = From f84b155c3e5cd5d60cb3f78224a117a273a849f4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jan 2020 18:24:21 -0600 Subject: [PATCH 14/22] ptr cchar to cstring, array support, named type procs --- nimterop/ast2.nim | 200 +++++++++++++++++++++++++++++++------------ nimterop/getters.nim | 29 +++++-- 2 files changed, 163 insertions(+), 66 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6f8f543..dc1a4b1 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -60,24 +60,51 @@ proc newTypeIdent(nimState: NimState, node: TSNode): PNode = result.add ident result.add newNode(nkEmpty) -proc newPtrTree(count: int): tuple[parent, child: PNode] = +proc newPtrTree(count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # + # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring + # # nkPtrTy( # nkPtrTy( - # .. + # typ # ) # ) + var + count = count + if typ.kind == nkIdent: + let + tname = typ.ident.s + ptname = getPtrType(tname) + if tname != ptname: + typ.ident.s = ptname + count -= 1 if count > 0: - result.parent = newNode(nkPtrTy) - result.child = result.parent + result = newNode(nkPtrTy) + var + parent = result + child: PNode for i in 1 ..< count: - let - child = newNode(nkPtrTy) - result.child.add child - result.child = child + child = newNode(nkPtrTy) + parent.add child + parent = child + parent.add typ + else: + result = typ -proc addType0(nimState: NimState, node: TSNode) = +proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = + result = newNode(nkBracketExpr) + + let + (_, info) = nimState.getNameInfo(node, nskType) + ident = nimState.getIdent("array", info, exported = false) + + result.add ident + result.add size + result.add typ + +proc addTypeObject(nimState: NimState, node: TSNode) = + # Add a type of object let typeDef = nimState.newTypeIdent(node) @@ -110,7 +137,8 @@ proc addType0(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc addType1(nimState: NimState, node: TSNode) = +proc addTypeTyped(nimState: NimState, node: TSNode) = + # Add a type of a specific type let # node[1] = identifer = name typeDef = nimState.newTypeIdent(node[1]) @@ -118,10 +146,9 @@ proc addType1(nimState: NimState, node: TSNode) = # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) # TODO - check blank and override - # TODO - use getPtrType() - ptr cchar to cstring ident = nimState.getIdent(name, info, exported = false) - # node[1] could have pointers + # node[1] could have nested pointers count = node[1].getPtrCount() # type X* = [ptr ..] Y @@ -137,13 +164,62 @@ proc addType1(nimState: NimState, node: TSNode) = # ) # ) - if count > 0: + # Skip typedef X X; + if $typeDef[0][1] != name: + if count > 0: + # If pointers + typeDef.add newPtrTree(count, ident) + else: + typeDef.add ident + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + +proc addTypeArray(nimState: NimState, node: TSNode) = + # Add a type of a array type + let + # node[1] = identifer = name + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = type name + (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) + # TODO - check blank and override + ident = nimState.getIdent(name, info, exported = false) + + # Top-most array declarator + adecl = node[1].firstChildInTree("array_declarator") + + # node[1] could have nested arrays + acount = adecl.getArrayCount() + innermost = adecl.mostNestedChildInTree() + + # node[1] could have nested pointers - type + tcount = node[1].getPtrCount() + + # Name could have nested pointers + ncount = innermost[0].getPtrCount() + + var + cnode = adecl + typ = ident + + if tcount > 0: + # If pointers + typ = newPtrTree(tcount, typ) + + for i in 0 ..< acount: let - (parent, child) = newPtrTree(count) - child.add ident - typeDef.add parent - else: - typeDef.add ident + size = nimState.getNodeVal(cnode[1]).getLit() + if size.kind != nkNilLit: + typ = nimState.newArrayTree(cnode, typ, size) + cnode = cnode[0] + + if ncount > 0: + typ = newPtrTree(ncount, typ) + + typeDef.add typ # nkTypeSection.add nimState.typeSection.add typeDef @@ -151,22 +227,6 @@ proc addType1(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = - # CASE1: - # - # - # CASE 2 - # - # typedef struct X *Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - # (type_identifier) - # ) - # ) - # nimState.printDebug(node) if node.getName() == "struct_specifier": @@ -176,7 +236,7 @@ proc addType(nimState: NimState, node: TSNode) = # (struct_specifier # (type_identifier) # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif node.len == 2: if node[1].getName() == "field_declaration_list" and node[1].len == 0: # struct X {}; @@ -185,7 +245,7 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # (field_declaration_list = "{}") # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif node.getName() == "type_definition": if node.len == 2: let @@ -200,7 +260,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) - nimState.addType0(node) + nimState.addTypeObject(node) elif fdlist.len == 0: # typedef struct X {}; # @@ -211,16 +271,17 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) - nimState.addType0(node) + nimState.addTypeObject(node) else: let sspec = node[0].firstChildInTree("struct_specifier") - adecl = node[1].anyChildInTree("array_declarator") fdecl = node[1].anyChildInTree("function_declarator") + adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): if adecl.isNil and fdecl.isNil: if not sspec.isNil: - # typedef struct X [*]Y; + # typedef struct X Y; + # typedef struct X *Y; # # (type_definition # (struct_specifier @@ -230,9 +291,10 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) - nimState.addType1(node) + nimState.addTypeTyped(node) else: - # typedef struct int Y; + # typedef X Y; + # typedef X *Y; # # (type_definition # (type_identifier|primitive_type) @@ -240,22 +302,46 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) - nimState.addType1(node) - elif not adecl.isNil: - # typedef X Y[a][..]; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - # (array_declarator - # (type_identifier) - # (number_literal) - # ) - # ) - # ) + nimState.addTypeTyped(node) + elif not fdecl.isNil: discard + elif not adecl.isNil: + if not sspec.isNil: + # typedef struct X Y[a][..]; + # typedef struct X *Y[a][..]; + # typedef struct X *(*Y)[a][..]; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) + else: + # typedef X Y[a][..]; + # typedef X *Y[a][..]; + # typedef X *(*Y)[a][..]; + # + # (type_definition + # (type_identifier|primitive_type) + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 2269268..46c333a 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -97,14 +97,10 @@ proc getType*(str: string): string = proc getPtrType*(str: string): string = result = case str: - of "ptr cchar": + of "cchar": "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": + of "object": "pointer" - of "ptr ptr object": - "ptr pointer" else: str @@ -267,18 +263,24 @@ proc getAtom*(node: TSNode): TSNode = elif node.len() != 0: return node[0].getAtom() -proc getPtrCount*(node: TSNode): int = +proc getXCount*(node: TSNode, ntype: string): int = if not node.isNil: - # Get number of ptr nodes in tree + # Get number of ntype nodes nested in tree var cnode = node - while "pointer_declarator" in cnode.getName(): + while ntype in cnode.getName(): result += 1 if cnode.len() != 0: cnode = cnode[0] else: break +proc getPtrCount*(node: TSNode): int = + node.getXCount("pointer_declarator") + +proc getArrayCount*(node: TSNode): int = + node.getXCount("array_declarator") + proc getDeclarator*(node: TSNode): TSNode = if not node.isNil: # Return if child is a function or array declarator @@ -310,6 +312,15 @@ proc anyChildInTree*(node: TSNode, ntype: string): TSNode = return ccnode cnode = cnode.tsNodeNextNamedSibling() +proc mostNestedChildInTree*(node: TSNode): TSNode = + # Search for the most nested child of node's type in tree + var + cnode = node + ntype = cnode.getName() + while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype: + cnode = cnode[0] + result = cnode + proc inChildren*(node: TSNode, ntype: string): bool = # Search for node type in immediate children result = false From 88917e952495e9714c94857e0d5a36ac2a35fba5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 4 Jan 2020 11:55:15 -0600 Subject: [PATCH 15/22] Upgrade tree-sitter version, collapse duplicate if, begin func type --- nimterop/ast2.nim | 153 ++++++++++++++++++++++++++------------------- nimterop/setup.nim | 6 +- 2 files changed, 92 insertions(+), 67 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dc1a4b1..9f64eef 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -106,6 +106,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = proc addTypeObject(nimState: NimState, node: TSNode) = # Add a type of object let + # TODO - check blank and override typeDef = nimState.newTypeIdent(node) # type X* = object @@ -141,11 +142,11 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # Add a type of a specific type let # node[1] = identifer = name + # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - # TODO - check blank and override ident = nimState.getIdent(name, info, exported = false) # node[1] could have nested pointers @@ -178,14 +179,14 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addTypeArray(nimState: NimState, node: TSNode) = - # Add a type of a array type + # Add a type of array type let # node[1] = identifer = name + # TODO - check blank and override typeDef = nimState.newTypeIdent(node[1]) # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - # TODO - check blank and override ident = nimState.getIdent(name, info, exported = false) # Top-most array declarator @@ -226,6 +227,27 @@ proc addTypeArray(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) +proc addTypeFunc(nimState: NimState, node: TSNode) = + # Add a type of function type + let + # node[1] = identifier = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + + # node[0] = identifier = return type name + (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + + # node[1] could have nested pointers + count = node[1].getArrayCount() + + # Parameter list + plist = node[1].anyChildInTree("parameter_list") + + var + retType = nimState.getIdent(rname, rinfo) + if count > 0: + retType = newPtrTree(count, retType) + proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -279,69 +301,72 @@ proc addType(nimState: NimState, node: TSNode) = adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): if adecl.isNil and fdecl.isNil: - if not sspec.isNil: - # typedef struct X Y; - # typedef struct X *Y; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # ) - nimState.addTypeTyped(node) - else: - # typedef X Y; - # typedef X *Y; - # - # (type_definition - # (type_identifier|primitive_type) - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # ) - nimState.addTypeTyped(node) + # typedef X Y; + # typedef X *Y; + # typedef struct X Y; + # typedef struct X *Y; + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + nimState.addTypeTyped(node) elif not fdecl.isNil: - discard + # typedef X (*Y)(a1, a2, a3); + # typedef X *(*Y)(a1, a2, a3); + # typedef struct X (*Y)(a1, a2, a3); + # typedef struct X *(*Y)(a1, a2, a3); + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator + # (type_identifer) + # ) + # ) + # (parameter_list + # (parameter_declaration + # (struct_specifier|type_identifier|primitive_type|array_declarator|function_declarator) + # (identifier - optional) + # ) + # ) + # ) + # ) + # ) + nimState.addTypeFunc(node) elif not adecl.isNil: - if not sspec.isNil: - # typedef struct X Y[a][..]; - # typedef struct X *Y[a][..]; - # typedef struct X *(*Y)[a][..]; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (pointer_declarator - optional, nested - # (array_declarator - nested - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # (number_literal) - # ) - # ) - # ) - nimState.addTypeArray(node) - else: - # typedef X Y[a][..]; - # typedef X *Y[a][..]; - # typedef X *(*Y)[a][..]; - # - # (type_definition - # (type_identifier|primitive_type) - # (pointer_declarator - optional, nested - # (array_declarator - nested - # (pointer_declarator - optional, nested - # (type_identifier) - # ) - # (number_literal) - # ) - # ) - # ) - nimState.addTypeArray(node) + # typedef struct X Y[a][..]; + # typedef struct X *Y[a][..]; + # typedef struct X *(*Y)[a][..]; + # + # (type_definition + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (pointer_declarator - optional, nested + # (array_declarator - nested + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # (number_literal) + # ) + # ) + # ) + nimState.addTypeArray(node) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 3f16dd7..cfa9a63 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -6,7 +6,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.5") +""", "0.16.2") gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ *.c @@ -36,7 +36,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.16.0") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -48,7 +48,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.16.0") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); From 8d3d41f5d8b1ff3de6e0fdbb335c0a4667062ff2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Jan 2020 01:51:15 -0600 Subject: [PATCH 16/22] Add proc type, reverse X count --- nimterop/ast2.nim | 262 ++++++++++++++++++++++++++++++++++++------- nimterop/getters.nim | 15 ++- 2 files changed, 232 insertions(+), 45 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 9f64eef..68956ae 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -93,6 +93,13 @@ proc newPtrTree(count: int, typ: PNode): PNode = result = typ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = + # Create nkBracketExpr tree depending on input + # + # nkBracketExpr( + # nkIdent("array"), + # size, + # typ + # ) result = newNode(nkBracketExpr) let @@ -103,6 +110,86 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add size result.add typ +proc getTypeArray(nimState: NimState, node: TSNode): PNode +proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode + +proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64): PNode = + # Create nkIdentDefs tree for specified parameter + result = newNode(nkIdentDefs) + + let + # node[0] - param type + (tname, tinfo) = nimState.getNameInfo(node[0], nskType) + tident = nimState.getIdent(tname, tinfo, exported = false) + + if node.len == 1: + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported = false) + result.add pident + result.add tident + result.add newNode(nkEmpty) + else: + let + fdecl = node[1].anyChildInTree("function_declarator") + adecl = node[1].anyChildInTree("array_declarator") + abst = node[1].getName() == "abstract_pointer_declarator" + if fdecl.isNil and adecl.isNil: + if abst: + let + pname = "a" & $(offset+1) + pident = nimState.getIdent(pname, tinfo, exported = false) + acount = node[1].getXCount("abstract_pointer_declarator") + result.add pident + result.add newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + + count = node[1].getPtrCount() + result.add pident + if count > 0: + result.add newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + result.add pident + result.add nimState.getTypeProc(node, name) + result.add newNode(nkEmpty) + elif not adecl.isNil: + let + (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + pident = nimState.getIdent(pname, pinfo, exported = false) + result.add pident + result.add nimState.getTypeArray(node) + result.add newNode(nkEmpty) + else: + result = nil + +proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + result = newNode(nkProcTy) + + let + fparam = newNode(nkFormalParams) + result.add fparam + result.add newNode(nkEmpty) + + # Add return type + fparam.add rtyp + + if not node.isNil: + for i in 0 ..< node.len: + let + a = nimState.newProcParam(name, node[i], i) + if not a.isNil: + fparam.add a + proc addTypeObject(nimState: NimState, node: TSNode) = # Add a type of object let @@ -152,19 +239,6 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # node[1] could have nested pointers count = node[1].getPtrCount() - # type X* = [ptr ..] Y - # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkPtrTy( # optional, nested - # nkIdent("Y") - # ) - # ) - # Skip typedef X X; if $typeDef[0][1] != name: if count > 0: @@ -173,18 +247,27 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = else: typeDef.add ident + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) -proc addTypeArray(nimState: NimState, node: TSNode) = - # Add a type of array type +proc getTypeArray(nimState: NimState, node: TSNode): PNode = + # Create array type tree let - # node[1] = identifer = name - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) - # node[0] = identifier = type name (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) ident = nimState.getIdent(name, info, exported = false) @@ -199,54 +282,156 @@ proc addTypeArray(nimState: NimState, node: TSNode) = # node[1] could have nested pointers - type tcount = node[1].getPtrCount() - # Name could have nested pointers - ncount = innermost[0].getPtrCount() + # Name could be nested pointer to array + # + # (.. + # (array_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = innermost[0].getAtom().tsNodeParent().getPtrCount(reverse = true) + result = ident var cnode = adecl - typ = ident if tcount > 0: # If pointers - typ = newPtrTree(tcount, typ) + result = newPtrTree(tcount, result) for i in 0 ..< acount: let size = nimState.getNodeVal(cnode[1]).getLit() if size.kind != nkNilLit: - typ = nimState.newArrayTree(cnode, typ, size) + result = nimState.newArrayTree(cnode, result, size) cnode = cnode[0] if ncount > 0: - typ = newPtrTree(ncount, typ) + result = newPtrTree(ncount, result) + +proc addTypeArray(nimState: NimState, node: TSNode) = + # Add a type of array type + let + # node[1] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + + typ = nimState.getTypeArray(node) typeDef.add typ + # type X* = [ptr] array[x, [ptr] Y] + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkBracketExpr( + # nkIdent("array") + # nkXLit(x), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + # ) + # ) + # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) -proc addTypeFunc(nimState: NimState, node: TSNode) = - # Add a type of function type +proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode = + # Create proc type tree let - # node[1] = identifier = name - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) - # node[0] = identifier = return type name (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) - # node[1] could have nested pointers - count = node[1].getArrayCount() - # Parameter list plist = node[1].anyChildInTree("parameter_list") + # node[1] could have nested pointers + tcount = node[1].getPtrCount() + + # Name could be nested pointer to function + # + # (.. + # (function_declarator + # (parenthesized_declarator + # (pointer_declarator .. + # (pointer_declarator <- search upwards from atom + # (type_identifier) <- atom + # ) + # ) + # ) + # ) + # ) + ncount = node[1].getAtom().tsNodeParent().getPtrCount(reverse = true) + + # Return type var - retType = nimState.getIdent(rname, rinfo) - if count > 0: - retType = newPtrTree(count, retType) + retType = nimState.getIdent(rname, rinfo, exported = false) + if tcount > 0: + retType = newPtrTree(tcount, retType) + + # Proc with return type and params + result = nimState.newProcTree(name, plist, retType) + if ncount > 1: + result = newPtrTree(ncount-1, result) + +proc addTypeProc(nimState: NimState, node: TSNode) = + # Add a type of proc type + let + # node[1] = identifier = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[1]) + name = $typeDef[0][1] + + procTy = nimState.getTypeProc(node, name) + + typeDef.add procTy + + # type X* = proc(a1: Y, a2: Z): P + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkProcTy( + # nkFormalParams( + # nkPtrTy( # optional, nested + # nkIdent(retType) + # ), + # nkIdentDefs( + # nkIdent(param), + # nkPtrTy( + # nkIdent(ptype) + # ), + # nkEmpty() + # ), + # ... + # ), + # nkEmpty() + # ) + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) @@ -296,7 +481,6 @@ proc addType(nimState: NimState, node: TSNode) = nimState.addTypeObject(node) else: let - sspec = node[0].firstChildInTree("struct_specifier") fdecl = node[1].anyChildInTree("function_declarator") adecl = node[1].anyChildInTree("array_declarator") if fdlist.isNil(): @@ -345,7 +529,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) - nimState.addTypeFunc(node) + nimState.addTypeProc(node) elif not adecl.isNil: # typedef struct X Y[a][..]; # typedef struct X *Y[a][..]; diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 46c333a..081364a 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -263,22 +263,25 @@ proc getAtom*(node: TSNode): TSNode = elif node.len() != 0: return node[0].getAtom() -proc getXCount*(node: TSNode, ntype: string): int = +proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if not node.isNil: # Get number of ntype nodes nested in tree var cnode = node while ntype in cnode.getName(): result += 1 - if cnode.len() != 0: - cnode = cnode[0] + if reverse: + cnode = cnode.tsNodeParent() else: - break + if cnode.len() != 0: + cnode = cnode[0] + else: + break -proc getPtrCount*(node: TSNode): int = +proc getPtrCount*(node: TSNode, reverse = false): int = node.getXCount("pointer_declarator") -proc getArrayCount*(node: TSNode): int = +proc getArrayCount*(node: TSNode, reverse = false): int = node.getXCount("array_declarator") proc getDeclarator*(node: TSNode): TSNode = From 166eae7a5a7bdc697c2a8103e89302f5415b5bad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 7 Jan 2020 01:12:42 -0600 Subject: [PATCH 17/22] Struct fields support, multiple bugs --- config.nims | 5 + nimterop/ast2.nim | 366 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 264 insertions(+), 107 deletions(-) diff --git a/config.nims b/config.nims index d312c07..e3d4ce0 100644 --- a/config.nims +++ b/config.nims @@ -13,3 +13,8 @@ switch("stackTrace", "on") # Path to compiler switch("path", "$nim") + +# Case objects +when not defined(danger): + switch("define", "nimOldCaseObjects") + diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 68956ae..caa2a13 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -49,12 +49,16 @@ proc addConst(nimState: NimState, node: TSNode) = nimState.printDebug(constDef) -proc newTypeIdent(nimState: NimState, node: TSNode): PNode = +proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = # Create nkTypeDef PNode with first ident let (name, info) = nimState.getNameInfo(node.getAtom(), nskType) # TODO - check blank and override - ident = nimState.getIdent(name, info) + ident = + if override.len != 0: + nimState.getIdent(override, info) + else: + nimState.getIdent(name, info) result = newNode(nkTypeDef) result.add ident @@ -111,10 +115,29 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = result.add typ proc getTypeArray(nimState: NimState, node: TSNode): PNode -proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode -proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64): PNode = - # Create nkIdentDefs tree for specified parameter +proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64, exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter or object field + # + # For proc, param should not be exported + # + # nkIdentDefs( + # nkIdent(pname), + # typ, + # nkEmpty() + # ) + # + # For object, field should be exported + # + # nkIdentDefs( + # nkPostfix( + # nkIdent("*"), + # nkIdent(pname) + # ), + # typ, + # nkEmpty() + # ) result = newNode(nkIdentDefs) let @@ -123,9 +146,12 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 tident = nimState.getIdent(tname, tinfo, exported = false) if node.len == 1: + # Only for proc with no named param - create a param name based on offset + # + # int func(char, int); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported = false) + pident = nimState.getIdent(pname, tinfo, exported) result.add pident result.add tident result.add newNode(nkEmpty) @@ -136,17 +162,22 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 abst = node[1].getName() == "abstract_pointer_declarator" if fdecl.isNil and adecl.isNil: if abst: + # Only for proc with no named param with pointer type + # Create a param name based on offset + # + # int func(char *, int **); let pname = "a" & $(offset+1) - pident = nimState.getIdent(pname, tinfo, exported = false) + pident = nimState.getIdent(pname, tinfo, exported) acount = node[1].getXCount("abstract_pointer_declarator") result.add pident result.add newPtrTree(acount, tident) result.add newNode(nkEmpty) else: + # Named param, simple type let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) count = node[1].getPtrCount() result.add pident @@ -156,16 +187,18 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 result.add tident result.add newNode(nkEmpty) elif not fdecl.isNil: + # Named param, function pointer let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) result.add pident - result.add nimState.getTypeProc(node, name) + result.add nimState.getTypeProc(name, node) result.add newNode(nkEmpty) elif not adecl.isNil: + # Named param, array type let (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) - pident = nimState.getIdent(pname, pinfo, exported = false) + pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeArray(node) result.add newNode(nkEmpty) @@ -173,81 +206,66 @@ proc newProcParam(nimState: NimState, name: string, node: TSNode, offset: uint64 result = nil proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = + # Create nkProcTy tree for specified proc type + # + # nkProcTy( + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ), + # nkEmpty() + # ) result = newNode(nkProcTy) let fparam = newNode(nkFormalParams) - result.add fparam - result.add newNode(nkEmpty) # Add return type fparam.add rtyp if not node.isNil: for i in 0 ..< node.len: + # Add nkIdentDefs for each param let - a = nimState.newProcParam(name, node[i], i) - if not a.isNil: - fparam.add a + param = nimState.newIdentDefs(name, node[i], i, exported = false) + if not param.isNil: + fparam.add param -proc addTypeObject(nimState: NimState, node: TSNode) = - # Add a type of object - let - # TODO - check blank and override - typeDef = nimState.newTypeIdent(node) + result.add fparam + result.add newNode(nkEmpty) - # type X* = object +proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = + # Create nkRecList tree for specified object # - # nkTypeDef( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkEmpty(), - # nkObjectTy( - # nkEmpty(), - # nkEmpty(), - # nkEmpty() + # nkRecList( + # nkIdentDefs( # multiple depending on fields + # .. # ) # ) - typeDef.add(block: - let - obj = newNode(nkObjectTy) - obj.add newNode(nkEmpty) - obj.add newNode(nkEmpty) - obj.add newNode(nkEmpty) + if not node.isNil: + result = newNode(nkRecList) - obj - ) + for i in 0 ..< node.len: + # Add nkIdentDefs for each field + let + field = nimState.newIdentDefs(name, node[i], i, exported = true) + if not field.isNil: + result.add field - # nkTypeSection.add - nimState.typeSection.add typeDef - - nimState.printDebug(typeDef) - -proc addTypeTyped(nimState: NimState, node: TSNode) = - # Add a type of a specific type +proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "") = + # Add a type of object + # + # If `override` is set, use it as the name + # If `duplicate` is set, don't add the same name let - # node[1] = identifer = name # TODO - check blank and override - typeDef = nimState.newTypeIdent(node[1]) + typeDef = nimState.newTypeIdent(node, override) + name = $typeDef[0][1] - # node[0] = identifier = type name - (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) - ident = nimState.getIdent(name, info, exported = false) - - # node[1] could have nested pointers - count = node[1].getPtrCount() - - # Skip typedef X X; - if $typeDef[0][1] != name: - if count > 0: - # If pointers - typeDef.add newPtrTree(count, ident) - else: - typeDef.add ident - - # type X* = [ptr ..] Y + if name != duplicate: + # type X* = object # # nkTypeDef( # nkPostfix( @@ -255,16 +273,83 @@ proc addTypeTyped(nimState: NimState, node: TSNode) = # nkIdent("X") # ), # nkEmpty(), - # nkPtrTy( # optional, nested - # nkIdent("Y") + # nkObjectTy( + # nkEmpty(), + # nkEmpty(), + # nkEmpty() # ) # ) + let + obj = newNode(nkObjectTy) + obj.add newNode(nkEmpty) + obj.add newNode(nkEmpty) + + let + fdlist = node.anyChildInTree("field_declaration_list") + if not fdlist.isNil and fdlist.len > 0: + # Add fields to object if present + obj.add nimState.newRecListTree(name, fdlist) + else: + obj.add newNode(nkEmpty) + + typeDef.add obj # nkTypeSection.add nimState.typeSection.add typeDef nimState.printDebug(typeDef) +proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = "") = + # Add a type of a specified type + # + # If `toverride` is set, use it as the type name + # If `duplicate` is set, don't add the same name + for i in 1 ..< node.len: + # Add a type of a specific type + let + # node[i] = identifer = name + # TODO - check blank and override + typeDef = nimState.newTypeIdent(node[i]) + + # node[0] = identifier = type name + (tname0, tinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + + # Override type name + tname = if toverride.len != 0: toverride else: tname0 + + ident = nimState.getIdent(tname, tinfo, exported = false) + + # node[i] could have nested pointers + count = node[i].getPtrCount() + + # Skip typedef X X; + if $typeDef[0][1] != tname: + if count > 0: + # If pointers + typeDef.add newPtrTree(count, ident) + else: + typeDef.add ident + + # type X* = [ptr ..] Y + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkEmpty(), + # nkPtrTy( # optional, nested + # nkIdent("Y") + # ) + # ) + + # nkTypeSection.add + nimState.typeSection.add typeDef + + nimState.printDebug(typeDef) + else: + nimState.addTypeObject(node, duplicate) + proc getTypeArray(nimState: NimState, node: TSNode): PNode = # Create array type tree let @@ -350,7 +435,7 @@ proc addTypeArray(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) -proc getTypeProc(nimState: NimState, node: TSNode, name: string): PNode = +proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = # Create proc type tree let # node[0] = identifier = return type name @@ -396,7 +481,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = typeDef = nimState.newTypeIdent(node[1]) name = $typeDef[0][1] - procTy = nimState.getTypeProc(node, name) + procTy = nimState.getTypeProc(name, node) typeDef.add procTy @@ -437,48 +522,59 @@ proc addType(nimState: NimState, node: TSNode) = nimState.printDebug(node) if node.getName() == "struct_specifier": - if node.len == 1: - # struct X; - # - # (struct_specifier - # (type_identifier) - # ) - nimState.addTypeObject(node) - elif node.len == 2: - if node[1].getName() == "field_declaration_list" and node[1].len == 0: - # struct X {}; - # - # (struct_specifier - # (type_identifier) - # (field_declaration_list = "{}") - # ) - nimState.addTypeObject(node) + # struct X; + # + # (struct_specifier + # (type_identifier) + # ) + # + # struct X {}; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # + # struct X { char *a1; }; + # + # (struct_specifier + # (type_identifier) + # (field_declaration_list + # (field_declaration + # (type_identifier|primitive_type|) + # (struct_specifier + # (type_identifier) + # ) + # + # (field_identifier) + # ) + # (field_declaration ...) + # ) + nimState.addTypeObject(node) elif node.getName() == "type_definition": - if node.len == 2: + if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") if nimState.getNodeVal(node[1]) == "": - if fdlist.isNil(): - # typedef struct X; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # ) - # (type_definition = "") - # ) - nimState.addTypeObject(node) - elif fdlist.len == 0: - # typedef struct X {}; - # - # (type_definition - # (struct_specifier - # (type_identifier) - # (field_declaration_list = "{}") - # ) - # (type_definition = "") - # ) - nimState.addTypeObject(node) + # typedef struct X; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # ) + # (type_definition = "") + # ) + # + # typedef struct X {}; + # + # (type_definition + # (struct_specifier + # (type_identifier) + # (field_declaration_list = "{}") + # ) + # (type_definition = "") + # ) + nimState.addTypeObject(node[0]) else: let fdecl = node[1].anyChildInTree("function_declarator") @@ -551,6 +647,62 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) nimState.addTypeArray(node) + else: + if node.firstChildInTree("field_declaration_list").isNil: + # typedef struct X { .. } Y, *Z; + # + # (type_definition + # (struct_specifier + # (type_identifier) - named struct <==== + # (field_declaration_list + # (field_declaration - optional, multiple + # (type_identifier|primitive_type|) + # (function_declarator|array_declarator + # .. + # ) + # + # (field_identifier) + # ) + # ) + # ) + # + # (type_identifier) + # (pointer_declarator - optional, nested + # (type_identifier) + # ) + # ) + + # First add struct as object + nimState.addTypeObject(node[0]) + + if node.len > 1 and nimState.getNodeVal(node[1]) != "": + # Add any additional names + nimState.addTypeTyped(node, duplicate = nimState.getNodeVal(node[0].getAtom())) + else: + # Same as above except unnamed struct + # + # typedef struct { .. } Y, *Z; + + # Get any name that isn't a pointer + let + name = block: + var + name = "" + for i in 1 ..< node.len: + if node[i].getName() == "type_identifier": + name = nimState.getNodeVal(node[i].getAtom()) + + if name.len == 0: + name = nimState.getUniqueIdentifier("STRUCT") + + name + + # Now add struct as object with specified name + nimState.addTypeObject(node[0], override = name) + + if node.len > 1 and nimState.getNodeVal(node[1]) != "": + # Add any additional names except duplicate + nimState.addTypeTyped(node, toverride = name, duplicate = name) proc addEnum(nimState: NimState, node: TSNode) = nimState.printDebug(node) From ee58be398ad47fe62a722d29bba9fc5344a513ad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 9 Jan 2020 13:58:34 -0600 Subject: [PATCH 18/22] Fix type bugs, enable basic tests --- nimterop.nimble | 23 ++++---- nimterop/ast2.nim | 128 +++++++++++++++++++++++++++--------------- nimterop/getters.nim | 9 ++- nimterop/globals.nim | 10 ++-- tests/include/test3.h | 42 ++++++++++++++ tests/tnimterop2.nim | 71 +++++++++++++++++++++++ 6 files changed, 223 insertions(+), 60 deletions(-) create mode 100644 tests/include/test3.h create mode 100644 tests/tnimterop2.nim diff --git a/nimterop.nimble b/nimterop.nimble index 3b6b7c1..39a2346 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -36,19 +36,22 @@ task docs, "Generate docs": task test, "Test": buildToastTask() - execTest "tests/tnimterop_c.nim" - execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" - execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" - execTest "tests/tpcre.nim" + execTest "tests/tnimterop2.nim" + + # Commented out until newalgo is ready + # execTest "tests/tnimterop_c.nim" + # execCmd "nim cpp -f -r tests/tnimterop_cpp.nim" + # execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h" + # execTest "tests/tpcre.nim" # Platform specific tests - when defined(Windows): - execTest "tests/tmath.nim" - if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): - execTest "tests/tsoloud.nim" + # when defined(Windows): + # execTest "tests/tmath.nim" + # if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): + # execTest "tests/tsoloud.nim" # getHeader tests - withDir("tests"): - execCmd("nim e getheader.nims") + # withDir("tests"): + # execCmd("nim e getheader.nims") docsTask() diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index caa2a13..6c62b05 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -13,6 +13,7 @@ proc addConst(nimState: NimState, node: TSNode) = # (identifier) # (preproc_arg) # ) + decho("addConst()") nimState.printDebug(node) if node[0].getName() == "identifier" and @@ -51,6 +52,8 @@ proc addConst(nimState: NimState, node: TSNode) = proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = # Create nkTypeDef PNode with first ident + # + # If `override`, use it instead of node.getAtom() for name let (name, info) = nimState.getNameInfo(node.getAtom(), nskType) # TODO - check blank and override @@ -60,30 +63,46 @@ proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode = else: nimState.getIdent(name, info) + # type name* = + # + # nkTypeDef( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkEmpty() + # ) result = newNode(nkTypeDef) result.add ident result.add newNode(nkEmpty) -proc newPtrTree(count: int, typ: PNode): PNode = +proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode = # Create nkPtrTy tree depending on count # # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring - # - # nkPtrTy( - # nkPtrTy( - # typ - # ) - # ) var count = count + chng = false if typ.kind == nkIdent: let tname = typ.ident.s ptname = getPtrType(tname) if tname != ptname: - typ.ident.s = ptname + # If Nim type available, use that ident + result = nimState.getIdent(ptname, typ.info, exported = false) + chng = true + # One ptr reduced count -= 1 if count > 0: + # Nested nkPtrTy(typ) depending on count + # + # [ptr ...] typ + # + # nkPtrTy( + # nkPtrTy( + # typ + # ) + # ) result = newNode(nkPtrTy) var parent = result @@ -93,11 +112,17 @@ proc newPtrTree(count: int, typ: PNode): PNode = parent.add child parent = child parent.add typ - else: + elif not chng: + # Either no ptr, or none left after Nim type adjustment result = typ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # Create nkBracketExpr tree depending on input + let + (_, info) = nimState.getNameInfo(node, nskType) + ident = nimState.getIdent("array", info, exported = false) + + # array[size, typ] # # nkBracketExpr( # nkIdent("array"), @@ -105,11 +130,6 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = # typ # ) result = newNode(nkBracketExpr) - - let - (_, info) = nimState.getNameInfo(node, nskType) - ident = nimState.getIdent("array", info, exported = false) - result.add ident result.add size result.add typ @@ -122,6 +142,8 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 # # For proc, param should not be exported # + # pname: [ptr ..] typ + # # nkIdentDefs( # nkIdent(pname), # typ, @@ -130,6 +152,8 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 # # For object, field should be exported # + # pname*: [ptr ..] typ + # # nkIdentDefs( # nkPostfix( # nkIdent("*"), @@ -171,7 +195,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 pident = nimState.getIdent(pname, tinfo, exported) acount = node[1].getXCount("abstract_pointer_declarator") result.add pident - result.add newPtrTree(acount, tident) + result.add nimState.newPtrTree(acount, tident) result.add newNode(nkEmpty) else: # Named param, simple type @@ -182,7 +206,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 count = node[1].getPtrCount() result.add pident if count > 0: - result.add newPtrTree(count, tident) + result.add nimState.newPtrTree(count, tident) else: result.add tident result.add newNode(nkEmpty) @@ -207,18 +231,6 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64 proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkProcTy tree for specified proc type - # - # nkProcTy( - # nkFormalParams( - # rtyp, - # nkIdentDefs( # multiple depending on params - # .. - # ) - # ), - # nkEmpty() - # ) - result = newNode(nkProcTy) - let fparam = newNode(nkFormalParams) @@ -233,18 +245,32 @@ proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): P if not param.isNil: fparam.add param + # proc(pname: ptyp ..): rtyp + # + # nkProcTy( + # nkFormalParams( + # rtyp, + # nkIdentDefs( # multiple depending on params + # .. + # ) + # ), + # nkEmpty() + # ) + result = newNode(nkProcTy) result.add fparam result.add newNode(nkEmpty) proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object - # - # nkRecList( - # nkIdentDefs( # multiple depending on fields - # .. - # ) - # ) if not node.isNil: + # fname*: ftyp + # .. + # + # nkRecList( + # nkIdentDefs( # multiple depending on fields + # .. + # ) + # ) result = newNode(nkRecList) for i in 0 ..< node.len: @@ -259,6 +285,7 @@ proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = # # If `override` is set, use it as the name # If `duplicate` is set, don't add the same name + decho("addTypeObject()") let # TODO - check blank and override typeDef = nimState.newTypeIdent(node, override) @@ -304,6 +331,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = # # If `toverride` is set, use it as the type name # If `duplicate` is set, don't add the same name + decho("addTypeTyped()") for i in 1 ..< node.len: # Add a type of a specific type let @@ -326,7 +354,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = if $typeDef[0][1] != tname: if count > 0: # If pointers - typeDef.add newPtrTree(count, ident) + typeDef.add nimState.newPtrTree(count, ident) else: typeDef.add ident @@ -348,7 +376,7 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = nimState.printDebug(typeDef) else: - nimState.addTypeObject(node, duplicate) + nimState.addTypeObject(node, duplicate = duplicate) proc getTypeArray(nimState: NimState, node: TSNode): PNode = # Create array type tree @@ -388,7 +416,7 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = if tcount > 0: # If pointers - result = newPtrTree(tcount, result) + result = nimState.newPtrTree(tcount, result) for i in 0 ..< acount: let @@ -398,10 +426,11 @@ proc getTypeArray(nimState: NimState, node: TSNode): PNode = cnode = cnode[0] if ncount > 0: - result = newPtrTree(ncount, result) + result = nimState.newPtrTree(ncount, result) proc addTypeArray(nimState: NimState, node: TSNode) = # Add a type of array type + decho("addTypeArray()") let # node[1] = identifer = name # TODO - check blank and override @@ -466,15 +495,16 @@ proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode = var retType = nimState.getIdent(rname, rinfo, exported = false) if tcount > 0: - retType = newPtrTree(tcount, retType) + retType = nimState.newPtrTree(tcount, retType) # Proc with return type and params result = nimState.newProcTree(name, plist, retType) if ncount > 1: - result = newPtrTree(ncount-1, result) + result = nimState.newPtrTree(ncount-1, result) proc addTypeProc(nimState: NimState, node: TSNode) = # Add a type of proc type + decho("addTypeProc()") let # node[1] = identifier = name # TODO - check blank and override @@ -519,6 +549,7 @@ proc addTypeProc(nimState: NimState, node: TSNode) = nimState.printDebug(typeDef) proc addType(nimState: NimState, node: TSNode) = + decho("addType()") nimState.printDebug(node) if node.getName() == "struct_specifier": @@ -550,12 +581,14 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (field_declaration ...) # ) + decho("addType(): case 1") nimState.addTypeObject(node) elif node.getName() == "type_definition": if node.len >= 2: let fdlist = node[0].anyChildInTree("field_declaration_list") - if nimState.getNodeVal(node[1]) == "": + if (fdlist.isNil or (not fdlist.isNil and fdlist.len == 0)) and + nimState.getNodeVal(node[1]) == "": # typedef struct X; # # (type_definition @@ -574,6 +607,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # (type_definition = "") # ) + decho("addType(): case 2") nimState.addTypeObject(node[0]) else: let @@ -596,6 +630,7 @@ proc addType(nimState: NimState, node: TSNode) = # (type_identifier) # ) # ) + decho("addType(): case 3") nimState.addTypeTyped(node) elif not fdecl.isNil: # typedef X (*Y)(a1, a2, a3); @@ -625,6 +660,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) + decho("addType(): case 4") nimState.addTypeProc(node) elif not adecl.isNil: # typedef struct X Y[a][..]; @@ -646,6 +682,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # ) # ) + decho("addType(): case 5") nimState.addTypeArray(node) else: if node.firstChildInTree("field_declaration_list").isNil: @@ -673,6 +710,7 @@ proc addType(nimState: NimState, node: TSNode) = # ) # First add struct as object + decho("addType(): case 6") nimState.addTypeObject(node[0]) if node.len > 1 and nimState.getNodeVal(node[1]) != "": @@ -684,6 +722,7 @@ proc addType(nimState: NimState, node: TSNode) = # typedef struct { .. } Y, *Z; # Get any name that isn't a pointer + decho("addType(): case 7") let name = block: var @@ -692,22 +731,21 @@ proc addType(nimState: NimState, node: TSNode) = if node[i].getName() == "type_identifier": name = nimState.getNodeVal(node[i].getAtom()) - if name.len == 0: - name = nimState.getUniqueIdentifier("STRUCT") - name # Now add struct as object with specified name nimState.addTypeObject(node[0], override = name) - if node.len > 1 and nimState.getNodeVal(node[1]) != "": + if name.len != 0: # Add any additional names except duplicate nimState.addTypeTyped(node, toverride = name, duplicate = name) proc addEnum(nimState: NimState, node: TSNode) = + decho("addEnum()") nimState.printDebug(node) proc addProc(nimState: NimState, node: TSNode) = + decho("addProc()") nimState.printDebug(node) proc processNode(nimState: NimState, node: TSNode): bool = diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 081364a..13093f7 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -313,7 +313,10 @@ proc anyChildInTree*(node: TSNode, ntype: string): TSNode = ccnode = cnode[i].anyChildInTree(ntype) if not ccnode.isNil(): return ccnode - cnode = cnode.tsNodeNextNamedSibling() + if cnode != node: + cnode = cnode.tsNodeNextNamedSibling() + else: + break proc mostNestedChildInTree*(node: TSNode): TSNode = # Search for the most nested child of node's type in tree @@ -444,6 +447,10 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = if offset.len == 0: echo "" +template decho*(str: untyped): untyped = + if nimState.gState.debug: + echo str.getCommented() + proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: echo ("Input => " & nimState.getNodeVal(node)).getCommented() diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 1921b1b..be24436 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,10 +1,11 @@ import sequtils, sets, tables -import compiler/[ast, idents, options] +import regex import "."/plugin when not declared(CIMPORT): + import compiler/[ast, idents, options] import "."/treesitter/api const @@ -52,9 +53,10 @@ type commentStr*, debugStr*, skipStr*: string # Nim compiler objects - constSection*, enumSection*, procSection*, typeSection*: PNode - identCache*: IdentCache - config*: ConfigRef + when not declared(CIMPORT): + constSection*, enumSection*, procSection*, typeSection*: PNode + identCache*: IdentCache + config*: ConfigRef gState*: State diff --git a/tests/include/test3.h b/tests/include/test3.h new file mode 100644 index 0000000..d1cf8aa --- /dev/null +++ b/tests/include/test3.h @@ -0,0 +1,42 @@ + +#define A 1 +#define B 1.0 +#define C 0x10 +#define D "hello" +#define E 'c' + +struct A0; +struct A1 {}; +typedef struct A2; +typedef struct A3 {}; +typedef struct A4 A4, *A4p; +typedef int A5; +typedef int *A6; +typedef A0 **A7; +typedef void *A8; + +typedef char *A9[3]; +typedef char *A10[3][6]; +typedef char *(*A11)[3]; + +typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); +typedef int A13(int, int); + +struct A14 { char a1; }; +struct A15 { char *a1; int *a2[1]; }; + +typedef struct A16 { char f1; }; +typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p; +typedef struct { char *a1; int *a2[1]; } A19, *A19p; + +typedef struct A20 { char a1; } A20, A21, *A21p; + +//Expression +//typedef struct A21 { int **f1; int abc[123+132]; } A21; + +//Unions +//union UNION1 {int f1; }; +//typedef union UNION2 { int **f1; int abc[123+132]; } UNION2; + +// Anonymous +//typedef struct { char a1; }; diff --git a/tests/tnimterop2.nim b/tests/tnimterop2.nim new file mode 100644 index 0000000..1e9c21d --- /dev/null +++ b/tests/tnimterop2.nim @@ -0,0 +1,71 @@ +import tables + +import nimterop/[cimport] + +static: + cDebug() + +cImport("include/test3.h", flags="-d") + +proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) = + var + obj: t + count = 0 + for name, value in obj.fieldPairs(): + count += 1 + assert name in fields, $t & "." & name & " invalid" + assert $fields[name] == $typeof(value), + "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value) + assert count == fields.len, "Failed for " & $t + +assert A == 1 +assert B == 1.0 +assert C == 0x10 +assert D == "hello" +assert E == 'c' + +assert A0 is object +testFields(A0) +assert A1 is object +testFields(A1) +assert A2 is object +testFields(A2) +assert A3 is object +testFields(A3) +assert A4 is object +testFields(A4) +assert A4p is ptr A4 +assert A5 is cint +assert A6 is ptr cint +assert A7 is ptr ptr A0 +assert A8 is pointer + +assert A9 is array[3, cstring] +assert A10 is array[3, array[6, cstring]] +assert A11 is ptr array[3, cstring] + +assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint +assert A13 is proc(a1: cint, a2: cint): cint + +assert A14 is object +testFields(A14, {"a1": "cchar"}.toTable()) + +assert A15 is object +testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) + +assert A16 is object +testFields(A16, {"f1": "cchar"}.toTable()) + +assert A17 is object +testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A18 is A17 +assert A18p is ptr A17 + +assert A19 is object +testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable()) +assert A19p is ptr A19 + +assert A20 is object +testFields(A20, {"a1": "cchar"}.toTable()) +assert A21 is A20 +assert A21p is ptr A20 From 1fe199e1d813fabde59d35065f94d7208df3c870 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 9 Jan 2020 17:50:09 -0600 Subject: [PATCH 19/22] Fix CI, drop 0.19.6 support --- nimterop/all.nim | 2 +- nimterop/ast2.nim | 4 ++-- nimterop/build.nim | 8 +++----- nimterop/cimport.nim | 2 +- nimterop/docs.nim | 30 +++++++++++++----------------- nimterop/getters.nim | 10 +++++----- nimterop/toast.nim | 2 +- nimterop/types.nim | 27 ++++----------------------- 8 files changed, 30 insertions(+), 55 deletions(-) diff --git a/nimterop/all.nim b/nimterop/all.nim index c024d01..dc22967 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -4,4 +4,4 @@ Module that should import everything so that `nim doc --project nimtero/all` run # TODO: make sure it does import everything. -import "."/[cimport, build, types, plugin, compat] +import "."/[cimport, build, types, plugin] diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 6c62b05..810beea 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -4,7 +4,7 @@ import compiler/[ast, astalgo, idents, options, renderer] import "."/treesitter/api -import "."/[compat, globals, getters] +import "."/[globals, getters] proc addConst(nimState: NimState, node: TSNode) = # #define X Y @@ -137,7 +137,7 @@ proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode = proc getTypeArray(nimState: NimState, node: TSNode): PNode proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode -proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: uint64, exported = false): PNode = +proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode = # Create nkIdentDefs tree for specified proc parameter or object field # # For proc, param should not be exported diff --git a/nimterop/build.nim b/nimterop/build.nim index 4b061cc..7939afa 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep -import "."/[compat] - proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: @@ -648,7 +646,7 @@ proc getGccPaths*(mode = "c"): seq[string] = break if inc: var - path = line.strip().myNormalizedPath() + path = line.strip().normalizedPath() if path notin result: result.add path @@ -667,13 +665,13 @@ proc getGccLibPaths*(mode = "c"): seq[string] = if "LIBRARY_PATH=" in line: for path in line[13 .. ^1].split(PathSep): var - path = path.strip().myNormalizedPath() + path = path.strip().normalizedPath() if path notin result: result.add path break elif '\t' in line: var - path = line.strip().myNormalizedPath() + path = line.strip().normalizedPath() if path notin result: result.add path diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 4c63c31..84f7b4f 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1 include "."/globals -import "."/[build, compat, paths, types] +import "."/[build, paths, types] export types proc interpPath(dir: string): string= diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 17089ec..7ea797a 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -1,22 +1,18 @@ import macros, strformat -when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): - from os import parentDir, getCurrentCompilerExe, DirSep - proc getNimRootDir(): string = - #[ - hack, but works - alternatively (but more complex), use (from a nim file, not nims otherwise - you get Error: ambiguous call; both system.fileExists): - import "$nim/testament/lib/stdtest/specialpaths.nim" - nimRootDir - ]# - fmt"{currentSourcePath}".parentDir.parentDir.parentDir -else: - proc getCurrentCompilerExe*(): string = - "nim" +from os import parentDir, getCurrentCompilerExe, DirSep +proc getNimRootDir(): string = + #[ + hack, but works + alternatively (but more complex), use (from a nim file, not nims otherwise + you get Error: ambiguous call; both system.fileExists): + import "$nim/testament/lib/stdtest/specialpaths.nim" + nimRootDir + ]# + fmt"{currentSourcePath}".parentDir.parentDir.parentDir - const - DirSep = when defined(windows): '\\' else: '/' +const + DirSep = when defined(windows): '\\' else: '/' proc execAction(cmd: string): string = var @@ -80,4 +76,4 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath for i in 0 .. paramCount(): if paramStr(i) == "--publish": echo execAction(&"cd {path} && ghp-import --no-jekyll -fp {path}") - break \ No newline at end of file + break diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 13093f7..1bb3efa 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -4,7 +4,7 @@ import regex import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer] -import "."/[build, compat, globals, plugin, treesitter/api] +import "."/[build, globals, plugin, treesitter/api] const gReserved = """ addr and as asm @@ -236,11 +236,11 @@ proc getLit*(str: string): PNode = proc isNil*(node: TSNode): bool = node.tsNodeIsNull() -proc len*(node: TSNode): uint = +proc len*(node: TSNode): int = if not node.isNil: - result = node.tsNodeNamedChildCount().uint + result = node.tsNodeNamedChildCount().int -proc `[]`*(node: TSNode, i: BiggestUInt): TSNode = +proc `[]`*(node: TSNode, i: SomeInteger): TSNode = if i < node.len(): result = node.tsNodeNamedChild(i.uint32) @@ -652,6 +652,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandSymlink().absolutePath(path.parentDir()).myNormalizedPath() + result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath() except: result = path diff --git a/nimterop/toast.nim b/nimterop/toast.nim index db68ce4..08f0dbb 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,7 @@ import os, osproc, strformat, strutils, times import "."/treesitter/[api, c, cpp] -import "."/[ast2, compat, globals, getters] +import "."/[ast2, globals, getters] proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/types.nim b/nimterop/types.nim index a57252f..89f779d 100644 --- a/nimterop/types.nim +++ b/nimterop/types.nim @@ -1,26 +1,7 @@ -# see https://github.com/nimterop/nimterop/issues/79 - -when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): - # clean this up once upgraded; adapted from std/time_t - when defined(nimdoc): - type - impl = distinct int64 - Time = impl - elif defined(windows): - when defined(i386) and defined(gcc): - type Time {.importc: "time_t", header: "".} = distinct int32 - else: - type Time {.importc: "time_t", header: "".} = distinct int64 - elif defined(posix): - import posix - type - time_t* = Time - time64_t* = Time -else: - import std/time_t as time_t_temp - type - time_t* = time_t_temp.Time - time64_t* = time_t_temp.Time +import std/time_t as time_t_temp +type + time_t* = time_t_temp.Time + time64_t* = time_t_temp.Time when defined(cpp): # http://www.cplusplus.com/reference/cwchar/wchar_t/ From c87587a068837c1475722bc4005a4022f61fdd2e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jan 2020 16:48:45 -0600 Subject: [PATCH 20/22] Fix bug --- nimterop/getters.nim | 4 ++-- nimterop/globals.nim | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 1bb3efa..42cb28b 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -453,8 +453,8 @@ template decho*(str: untyped): untyped = proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: - echo ("Input => " & nimState.getNodeVal(node)).getCommented() - echo nimState.gState.printLisp(node).getCommented() + necho ("Input => " & nimState.getNodeVal(node)).getCommented() & "\n" & + nimState.gState.printLisp(node).getCommented() proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index be24436..7dd6fe7 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -96,3 +96,7 @@ when not declared(CIMPORT): template necho*(args: string) {.dirty.} = let gState = nimState.gState gecho args + + template decho*(str: untyped): untyped = + if nimState.gState.debug: + necho str.getCommented() From 5241aee0fcb1a6a75b38ef1a73ed9b0f9113f2d7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jan 2020 12:49:37 -0600 Subject: [PATCH 21/22] Merge PR#166 from master --- nimterop/ast2.nim | 8 ++++---- nimterop/getters.nim | 8 ++------ nimterop/toast.nim | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 810beea..e6f3a63 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,6 +1,6 @@ import macros, os, strutils, tables, times -import compiler/[ast, astalgo, idents, options, renderer] +import compiler/[ast, idents, options, renderer] import "."/treesitter/api @@ -806,8 +806,8 @@ proc searchTree(nimState: NimState, root: TSNode) = if node == root: break -proc printNimHeader*() = - echo """# Generated at $1 +proc printNimHeader*(gState: State) = + gecho """# Generated at $1 # Command line: # $2 $3 @@ -845,4 +845,4 @@ proc printNim*(gState: State, fullpath: string, root: TSNode) = tree.add nimState.typeSection tree.add nimState.procSection - echo tree.renderTree() + gecho tree.renderTree() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 42cb28b..be58a48 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -445,11 +445,7 @@ proc printTree*(nimState: NimState, pnode: PNode, offset = "") = else: stdout.write ")" if offset.len == 0: - echo "" - -template decho*(str: untyped): untyped = - if nimState.gState.debug: - echo str.getCommented() + necho "" proc printDebug*(nimState: NimState, node: TSNode) = if nimState.gState.debug: @@ -458,7 +454,7 @@ proc printDebug*(nimState: NimState, node: TSNode) = proc printDebug*(nimState: NimState, pnode: PNode) = if nimState.gState.debug: - echo ("Output => " & $pnode).getCommented() + necho ("Output => " & $pnode).getCommented() nimState.printTree(pnode) # Compiler shortcuts diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 08f0dbb..3761d3e 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -42,7 +42,7 @@ proc process(gState: State, path: string) = tree.tsTreeDelete() if gState.past: - echo gState.printLisp(root) + gecho gState.printLisp(root) elif gState.pnim: gState.printNim(path, root) elif gState.preprocess: From 714fbe851fd7e970d7ba5d7b8a4cfd80b58e205c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 23 Feb 2020 22:40:05 -0600 Subject: [PATCH 22/22] Support type qualifiers --- nimterop.nimble | 2 +- nimterop/ast2.nim | 53 +++++++++++++++++++++++++------------------ nimterop/getters.nim | 9 ++++++++ tests/include/test3.h | 6 ++--- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 39a2346..11eb4ad 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -25,7 +25,7 @@ proc execTest(test: string) = execCmd "nim cpp -r " & test task buildToast, "build toast": - execCmd("nim c -f -d:danger nimterop/toast.nim") + execCmd("nim c -f nimterop/toast.nim") task bt, "build toast": execCmd("nim c -d:danger nimterop/toast.nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index e6f3a63..b14cf13 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,4 +1,4 @@ -import macros, os, strutils, tables, times +import macros, os, sets, strutils, tables, times import compiler/[ast, idents, options, renderer] @@ -165,11 +165,13 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn result = newNode(nkIdentDefs) let - # node[0] - param type - (tname, tinfo) = nimState.getNameInfo(node[0], nskType) + start = getStartAtom(node) + + # node[start] - param type + (tname, tinfo) = nimState.getNameInfo(node[start], nskType) tident = nimState.getIdent(tname, tinfo, exported = false) - if node.len == 1: + if start == node.len - 1: # Only for proc with no named param - create a param name based on offset # # int func(char, int); @@ -181,9 +183,9 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn result.add newNode(nkEmpty) else: let - fdecl = node[1].anyChildInTree("function_declarator") - adecl = node[1].anyChildInTree("array_declarator") - abst = node[1].getName() == "abstract_pointer_declarator" + fdecl = node[start+1].anyChildInTree("function_declarator") + adecl = node[start+1].anyChildInTree("array_declarator") + abst = node[start+1].getName() == "abstract_pointer_declarator" if fdecl.isNil and adecl.isNil: if abst: # Only for proc with no named param with pointer type @@ -193,17 +195,17 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn let pname = "a" & $(offset+1) pident = nimState.getIdent(pname, tinfo, exported) - acount = node[1].getXCount("abstract_pointer_declarator") + acount = node[start+1].getXCount("abstract_pointer_declarator") result.add pident result.add nimState.newPtrTree(acount, tident) result.add newNode(nkEmpty) else: # Named param, simple type let - (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) - count = node[1].getPtrCount() + count = node[start+1].getPtrCount() result.add pident if count > 0: result.add nimState.newPtrTree(count, tident) @@ -213,7 +215,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn elif not fdecl.isNil: # Named param, function pointer let - (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeProc(name, node) @@ -221,7 +223,7 @@ proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeIn elif not adecl.isNil: # Named param, array type let - (pname, pinfo) = nimState.getNameInfo(node[1].getAtom(), nskField, parent = name) + (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name) pident = nimState.getIdent(pname, pinfo, exported) result.add pident result.add nimState.getTypeArray(node) @@ -332,15 +334,17 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = # If `toverride` is set, use it as the type name # If `duplicate` is set, don't add the same name decho("addTypeTyped()") - for i in 1 ..< node.len: + let + start = getStartAtom(node) + for i in start+1 ..< node.len: # Add a type of a specific type let # node[i] = identifer = name # TODO - check blank and override typeDef = nimState.newTypeIdent(node[i]) - # node[0] = identifier = type name - (tname0, tinfo) = nimState.getNameInfo(node[0].getAtom(), nskType) + # node[start] = identifier = type name + (tname0, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType) # Override type name tname = if toverride.len != 0: toverride else: tname0 @@ -381,19 +385,21 @@ proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = proc getTypeArray(nimState: NimState, node: TSNode): PNode = # Create array type tree let - # node[0] = identifier = type name - (name, info) = nimState.getNameInfo(node[0].getAtom(), nskType) + start = getStartAtom(node) + + # node[start] = identifier = type name + (name, info) = nimState.getNameInfo(node[start].getAtom(), nskType) ident = nimState.getIdent(name, info, exported = false) # Top-most array declarator - adecl = node[1].firstChildInTree("array_declarator") + adecl = node[start+1].firstChildInTree("array_declarator") - # node[1] could have nested arrays + # node[start+1] could have nested arrays acount = adecl.getArrayCount() innermost = adecl.mostNestedChildInTree() - # node[1] could have nested pointers - type - tcount = node[1].getPtrCount() + # node[start+1] could have nested pointers - type + tcount = node[start+1].getPtrCount() # Name could be nested pointer to array # @@ -621,6 +627,7 @@ proc addType(nimState: NimState, node: TSNode) = # typedef struct X *Y; # # (type_definition + # (type_qualifier?) # (type_identifier|primitive_type|) # (struct_specifier # (type_identifier) @@ -639,6 +646,7 @@ proc addType(nimState: NimState, node: TSNode) = # typedef struct X *(*Y)(a1, a2, a3); # # (type_definition + # (type_qualifier?) # (type_identifier|primitive_type|) # (struct_specifier # (type_identifier) @@ -668,6 +676,7 @@ proc addType(nimState: NimState, node: TSNode) = # typedef struct X *(*Y)[a][..]; # # (type_definition + # (type_qualifier?) # (type_identifier|primitive_type|) # (struct_specifier # (type_identifier) @@ -819,7 +828,7 @@ import nimterop/types proc printNim*(gState: State, fullpath: string, root: TSNode) = var nimState = new(NimState) - fp = fullpath.replace("\\", "/") + #fp = fullpath.replace("\\", "/") nimState.identifiers = newTable[string, string]() diff --git a/nimterop/getters.nim b/nimterop/getters.nim index d56dd61..116c395 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -265,6 +265,15 @@ proc getAtom*(node: TSNode): TSNode = elif node.len() != 0: return node[0].getAtom() +proc getStartAtom*(node: TSNode): int = + if not node.isNil: + # Skip const, volatile and other type qualifiers + for i in 0 .. node.len - 1: + if node[i].getAtom().getName() notin gAtoms: + result += 1 + else: + break + proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if not node.isNil: # Get number of ntype nodes nested in tree diff --git a/tests/include/test3.h b/tests/include/test3.h index d1cf8aa..1a2789b 100644 --- a/tests/include/test3.h +++ b/tests/include/test3.h @@ -10,7 +10,7 @@ struct A1 {}; typedef struct A2; typedef struct A3 {}; typedef struct A4 A4, *A4p; -typedef int A5; +typedef const int A5; typedef int *A6; typedef A0 **A7; typedef void *A8; @@ -22,8 +22,8 @@ typedef char *(*A11)[3]; typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)); typedef int A13(int, int); -struct A14 { char a1; }; -struct A15 { char *a1; int *a2[1]; }; +struct A14 { volatile char a1; }; +struct A15 { char *a1; const int *a2[1]; }; typedef struct A16 { char f1; }; typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p;