Switch to simpler grammar engine

nimterop/ast.nim

@@ -1,265 +1,83 @@
-import macros, os, strformat, strutils
+import strformat, strutils, tables
+
+import regex
 
 import treesitter/runtime
 
-import getters, globals
+import "."/[getters, globals, grammar]
 
-#
+const gAtoms = @[
-# Preprocessor
+  "field_identifier",
-#
+  "identifier",
+  "number_literal",
+  "preproc_arg",
+  "primitive_type",
+  "type_identifier"
+]
 
-proc pPreprocDef(node: TSNode) =
+proc saveNodeData(node: TSNode): bool =
-  if node.tsNodeNamedChildCount() == 2:
+  let name = $node.tsNodeType()
-    let
+  if name in gAtoms:
-      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
-      val = getNodeValIf(node.tsNodeNamedChild(1), "preproc_arg")
-
-    if name.nBl and val.nBl and name notin gStateRT.consts:
-      gStateRT.consts.add(name)
-      if val.getLit().nBl:
-        # #define NAME VALUE
-        gStateRT.constStr &= &"  {name.getIdentifier()}* = {val}        # pPreprocDef()\n"
-
-#
-# Types
-#
-
-proc typeScan(node: TSNode, sym, id: string, offset: string): string =
-  if node.tsNodeIsNull() or $node.tsNodeType() != sym or node.tsNodeNamedChildCount() != 2:
-    return
-
-  var
-    name = getNodeValIf(node.tsNodeNamedChild(1), id)
-    ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-    ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    ptrname = false
-
-  if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
-    name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), id)
-    ptrname = true
-
-  if name.len == 0:
-    return
-  elif ptyp.nBl:
-    ptyp = ptyp.getType()
-    if ptyp != "object" and ptrname:
-      ptyp = &"ptr {ptyp}"
-    result = &"{offset}{name.getIdentifier()}: {ptyp}"
-  elif ttyp.nBl:
-    if ptrname:
-      ttyp = &"ptr {ttyp}"
-    result = &"{offset}{name.getIdentifier()}: {ttyp}"
-  elif $node.tsNodeNamedChild(0).tsNodeType() in ["struct_specifier", "enum_specifier"] and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
-    var styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
-    if styp.nBl:
-      if ptrname:
-        styp = &"ptr {styp}"
-      result = &"{offset}{name.getIdentifier()}: {styp}"
-
-proc pStructSpecifier(node: TSNode, name = "") =
-  var stmt: string
-  if node.tsNodeNamedChildCount() == 1 and name notin gStateRT.types:
-    case $node.tsNodeNamedChild(0).tsNodeType():
-      of "type_identifier":
-        let typ = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-        if typ.nBl:
-          gStateRT.types.add(name)
-          if name != typ:
-            # typedef struct X Y
-            gStateRT.typeStr &= &"  {name.getIdentifier()}* = {typ}        #1 pStructSpecifier()\n"
-          else:
-            # typedef struct X X
-            gStateRT.typeStr &= &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #2 pStructSpecifier()\n"
-
-      of "field_declaration_list":
-        # typedef struct { fields } X
-        stmt = &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #3 pStructSpecifier()\n"
-
-        if node.tsNodeNamedChild(0).tsNodeNamedChildCount() != 0:
-          for i in 0 .. node.tsNodeNamedChild(0).tsNodeNamedChildCount()-1:
-            if $node.tsNodeNamedChild(0).tsNodeNamedChild(i).tsNodeType() == "comment":
-              continue
-            let ts = typeScan(node.tsNodeNamedChild(0).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
-            if ts.len == 0:
-              return
-            stmt &= ts & "\n"
-
-          gStateRT.types.add(name)
-          gStateRT.typeStr &= stmt
-      else:
-        discard
-
-  elif name.len == 0 and node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "field_declaration_list":
-    let ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    if ename.nBl and ename notin gStateRT.types:
-      # struct X { fields }
-      stmt &= &"  {ename}* {{.importc: \"struct {ename}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #4 pStructSpecifier()\n"
-
-      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
-        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
-          if $node.tsNodeNamedChild(1).tsNodeNamedChild(i).tsNodeType() == "comment":
-            continue
-          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
-          if ts.len == 0:
-            return
-          stmt &= ts & "\n"
-
-        gStateRT.types.add(name)
-        gStateRT.typeStr &= stmt
-
-proc pEnumSpecifier(node: TSNode, name = "") =
-  var
-    ename: string
-    elid: uint32
-    stmt: string
-
-  if node.tsNodeNamedChildCount() == 1 and $node.tsNodeNamedChild(0).tsNodeType() == "enumerator_list":
-    # typedef enum { fields } X
-    ename = name
-    elid = 0
-    stmt = &"  {name.getIdentifier()}* = enum        #1 pEnumSpecifier()\n"
-  elif node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "enumerator_list":
-    if name.len == 0:
-      ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    else:
-      ename = name
-    elid = 1
-    if ename.nBl:
-      # enum X { fields }
-      stmt = &"  {ename}* = enum        #2 pEnumSpecifier()\n"
-    else:
-      return
-  else:
-    return
-
-  if node.tsNodeNamedChild(elid).tsNodeNamedChildCount() != 0:
-    for i in 0 .. node.tsNodeNamedChild(elid).tsNodeNamedChildCount()-1:
-      let field = node.tsNodeNamedChild(elid).tsNodeNamedChild(i)
-      if $field.tsNodeType() == "comment":
-        continue
-      if not field.tsNodeIsNull() and $field.tsNodeType() == "enumerator":
-        let fname = getNodeValIf(field.tsNodeNamedChild(0), "identifier")
-        if field.tsNodeNamedChildCount() == 1:
-          stmt &= &"    {fname}\n"
-        elif field.tsNodeNamedChildCount() == 2 and $field.tsNodeNamedChild(1).tsNodeType() == "number_literal":
-          let num = getNodeValIf(field.tsNodeNamedChild(1), "number_literal")
-          stmt &= &"    {fname} = {num}\n"
-        else:
-          return
-
-    if ename notin gStateRT.types:
-      gStateRT.types.add(name)
-      gStateRT.typeStr &= stmt
-
-proc pTypeDefinition(node: TSNode) =
-  if node.tsNodeNamedChildCount() == 2:
     var
-      name = getNodeValIf(node.tsNodeNamedChild(1), "type_identifier")
+      val = node.getNodeVal()
-      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-      ptrname = false
 
-    if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
+    if name == "primitive_type":
-      name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), "type_identifier")
+      val = val.getType()
-      ptrname = true
 
-    if name.nBl and name notin gStateRT.types:
+    if node.tsNodeParent().tsNodeType() == "pointer_declarator":
-      if ptyp.nBl:
+      if gStateRT.data[^1].val != "object":
-        # typedef int X
+        gStateRT.data[^1].val = "ptr " & gStateRT.data[^1].val
-        gStateRT.types.add(name)
-        ptyp = ptyp.getType()
-        if ptyp != "object" and ptrname:
-          ptyp = &"ptr {ptyp}"
-        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ptyp}        #1 pTypeDefinition()\n"
-      elif ttyp.nBl:
-        # typedef X Y
-        gStateRT.types.add(name)
-        if ptrname:
-          ttyp = &"ptr {ttyp}"
-        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ttyp}        #2 pTypeDefinition()\n"
-      else:
-        case $node.tsNodeNamedChild(0).tsNodeType():
-          of "struct_specifier":
-            pStructSpecifier(node.tsNodeNamedChild(0), name)
-          of "enum_specifier":
-            pEnumSpecifier(node.tsNodeNamedChild(0), name)
-          else:
-            discard
 
-proc pFunctionDeclarator(node: TSNode, typ: string) =
+    gStateRT.data.add((name, val))
-  if node.tsNodeNamedChildCount() == 2:
+
+  return true
+
+proc searchAstForNode(ast: ref Ast, node: TSNode): bool =
+  let
+    childNames = node.getTSNodeNamedChildNames().join()
+
+  if ast.children.len != 0:
     let
-      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
+      rstr = ast.getRegexForAstChildren()
 
-    if name.nBl and name notin gStateRT.procs and $node.tsNodeNamedChild(1).tsNodeType() == "parameter_list":
+    if childNames.contains(rstr.toPattern):
-      # typ function(typ param1, ...)
+      if node.getTSNodeNamedChildCountSansComments() != 0:
-      var stmt = &"# pFunctionDeclarator()\nproc {name.getIdentifier()}*("
+        var flag = true
+        for i in 0 .. node.tsNodeNamedChildCount()-1:
+          if node.tsNodeType() != "comment":
+            let
+              nodeChild = node.tsNodeNamedChild(i)
+              astChild = ast.getAstChildByName($nodeChild.tsNodeType())
+            if not searchAstForNode(astChild, nodeChild):
+              flag = false
+              break
 
-      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
+        if flag:
-        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
+          return node.saveNodeData()
-          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "parameter_declaration", "identifier", "")
-          if ts.len == 0:
-            return
-          stmt &= ts
-          if i != node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
-            stmt &= ", "
-
-      if typ != "void":
-        stmt &= &"): {typ.getType()} "
       else:
-        stmt &= ") "
+        return node.saveNodeData()
+  elif node.getTSNodeNamedChildCountSansComments() == 0:
+    return node.saveNodeData()
 
-      stmt &= &"{{.importc: \"{name}\", header: {gStateRT.currentHeader}.}}\n"
+proc searchAst(root: TSNode) =
-
-      gStateRT.procs.add(name)
-      gStateRT.procStr &= stmt
-
-proc pDeclaration*(node: TSNode) =
-  if node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "function_declarator":
-    let
-      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-
-    if ptyp.nBl:
-      pFunctionDeclarator(node.tsNodeNamedChild(1), ptyp.getType())
-    elif ttyp.nBl:
-      pFunctionDeclarator(node.tsNodeNamedChild(1), ttyp)
-    elif $node.tsNodeNamedChild(0).tsNodeType() == "struct_specifier" and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
-      let styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
-      if styp.nBl:
-        pFunctionDeclarator(node.tsNodeNamedChild(1), styp)
-
-proc genNimAst(root: TSNode) =
   var
     node = root
     nextnode: TSNode
 
   while true:
     if not node.tsNodeIsNull():
-      case $node.tsNodeType():
+      let
-        of "ERROR":
+        name = $node.tsNodeType()
-          let (line, col) = getLineCol(node)
+      if name in gStateRT.ast:
-          let file = gStateRT.sourceFile
+        for ast in gStateRT.ast[name]:
-          echo &"# [toast] Potentially invalid syntax at {file}:{line}:{col}"
+          if searchAstForNode(ast, node):
-        of "preproc_def":
+            ast.tonim()
-          pPreprocDef(node)
+            break
-        of "type_definition":
+        gStateRT.data = @[]
-          pTypeDefinition(node)
-        of "declaration":
-          pDeclaration(node)
-        of "struct_specifier":
-          if $node.tsNodeParent().tsNodeType() notin ["type_definition", "declaration"]:
-            pStructSpecifier(node)
-        of "enum_specifier":
-          if $node.tsNodeParent.tsNodeType() notin ["type_definition", "declaration"]:
-            pEnumSpecifier(node)
-        else:
-          # TODO: log
-          discard
     else:
       return
 
-    if node.tsNodeNamedChildCount() != 0:
+    if $node.tsNodeType() notin gStateRT.ast and node.tsNodeNamedChildCount() != 0:
       nextnode = node.tsNodeNamedChild(0)
     else:
       nextnode = node.tsNodeNextNamedSibling()
@@ -279,13 +97,15 @@ proc genNimAst(root: TSNode) =
       break
 
 proc printNim*(fullpath: string, root: TSNode) =
+  parseGrammar()
+
   echo "{.experimental: \"codeReordering\".}"
 
   var fp = fullpath.replace("\\", "/")
   gStateRT.currentHeader = getCurrentHeader(fullpath)
   gStateRT.constStr &= &"  {gStateRT.currentHeader} = \"{fp}\"\n"
 
-  genNimAst(root)
+  root.searchAst()
 
   if gStateRT.constStr.nBl:
     echo "const\n" & gStateRT.constStr

nimterop/astold.nim

@@ -0,0 +1,297 @@
+import macros, os, strformat, strutils
+
+import treesitter/runtime
+
+import getters, globals
+
+#
+# Preprocessor
+#
+
+proc pPreprocDef(node: TSNode) =
+  if node.tsNodeNamedChildCount() == 2:
+    let
+      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
+      val = getNodeValIf(node.tsNodeNamedChild(1), "preproc_arg")
+
+    if name.nBl and val.nBl and name notin gStateRT.consts:
+      gStateRT.consts.add(name)
+      if val.getLit().nBl:
+        # #define NAME VALUE
+        gStateRT.constStr &= &"  {name.getIdentifier()}* = {val}        # pPreprocDef()\n"
+
+#
+# Types
+#
+
+proc typeScan(node: TSNode, sym, id: string, offset: string): string =
+  if node.tsNodeIsNull() or $node.tsNodeType() != sym or node.tsNodeNamedChildCount() != 2:
+    return
+
+  var
+    name = getNodeValIf(node.tsNodeNamedChild(1), id)
+    ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
+    ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+    ptrname = false
+
+  if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
+    name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), id)
+    ptrname = true
+
+  if name.len == 0:
+    return
+  elif ptyp.nBl:
+    ptyp = ptyp.getType()
+    if ptyp != "object" and ptrname:
+      ptyp = &"ptr {ptyp}"
+    result = &"{offset}{name.getIdentifier()}: {ptyp}"
+  elif ttyp.nBl:
+    if ptrname:
+      ttyp = &"ptr {ttyp}"
+    result = &"{offset}{name.getIdentifier()}: {ttyp}"
+  elif $node.tsNodeNamedChild(0).tsNodeType() in ["struct_specifier", "enum_specifier"] and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
+    var styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
+    if styp.nBl:
+      if ptrname:
+        styp = &"ptr {styp}"
+      result = &"{offset}{name.getIdentifier()}: {styp}"
+
+proc pStructSpecifier(node: TSNode, name = "") =
+  var stmt: string
+  if node.tsNodeNamedChildCount() == 1 and name notin gStateRT.types:
+    case $node.tsNodeNamedChild(0).tsNodeType():
+      of "type_identifier":
+        let typ = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+        if typ.nBl:
+          gStateRT.types.add(name)
+          if name != typ:
+            # typedef struct X Y
+            gStateRT.typeStr &= &"  {name.getIdentifier()}* = {typ}        #1 pStructSpecifier()\n"
+          else:
+            # typedef struct X X
+            gStateRT.typeStr &= &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #2 pStructSpecifier()\n"
+
+      of "field_declaration_list":
+        # typedef struct { fields } X
+        stmt = &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #3 pStructSpecifier()\n"
+
+        if node.tsNodeNamedChild(0).tsNodeNamedChildCount() != 0:
+          for i in 0 .. node.tsNodeNamedChild(0).tsNodeNamedChildCount()-1:
+            if $node.tsNodeNamedChild(0).tsNodeNamedChild(i).tsNodeType() == "comment":
+              continue
+            let ts = typeScan(node.tsNodeNamedChild(0).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
+            if ts.len == 0:
+              return
+            stmt &= ts & "\n"
+
+          gStateRT.types.add(name)
+          gStateRT.typeStr &= stmt
+      else:
+        discard
+
+  elif name.len == 0 and node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "field_declaration_list":
+    let ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+    if ename.nBl and ename notin gStateRT.types:
+      # struct X { fields }
+      stmt &= &"  {ename}* {{.importc: \"struct {ename}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #4 pStructSpecifier()\n"
+
+      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
+        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
+          if $node.tsNodeNamedChild(1).tsNodeNamedChild(i).tsNodeType() == "comment":
+            continue
+          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
+          if ts.len == 0:
+            return
+          stmt &= ts & "\n"
+
+        gStateRT.types.add(name)
+        gStateRT.typeStr &= stmt
+
+proc pEnumSpecifier(node: TSNode, name = "") =
+  var
+    ename: string
+    elid: uint32
+    stmt: string
+
+  if node.tsNodeNamedChildCount() == 1 and $node.tsNodeNamedChild(0).tsNodeType() == "enumerator_list":
+    # typedef enum { fields } X
+    ename = name
+    elid = 0
+    stmt = &"  {name.getIdentifier()}* = enum        #1 pEnumSpecifier()\n"
+  elif node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "enumerator_list":
+    if name.len == 0:
+      ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+    else:
+      ename = name
+    elid = 1
+    if ename.nBl:
+      # enum X { fields }
+      stmt = &"  {ename}* = enum        #2 pEnumSpecifier()\n"
+    else:
+      return
+  else:
+    return
+
+  if node.tsNodeNamedChild(elid).tsNodeNamedChildCount() != 0:
+    for i in 0 .. node.tsNodeNamedChild(elid).tsNodeNamedChildCount()-1:
+      let field = node.tsNodeNamedChild(elid).tsNodeNamedChild(i)
+      if $field.tsNodeType() == "comment":
+        continue
+      if not field.tsNodeIsNull() and $field.tsNodeType() == "enumerator":
+        let fname = getNodeValIf(field.tsNodeNamedChild(0), "identifier")
+        if field.tsNodeNamedChildCount() == 1:
+          stmt &= &"    {fname}\n"
+        elif field.tsNodeNamedChildCount() == 2 and $field.tsNodeNamedChild(1).tsNodeType() == "number_literal":
+          let num = getNodeValIf(field.tsNodeNamedChild(1), "number_literal")
+          stmt &= &"    {fname} = {num}\n"
+        else:
+          return
+
+    if ename notin gStateRT.types:
+      gStateRT.types.add(name)
+      gStateRT.typeStr &= stmt
+
+proc pTypeDefinition(node: TSNode) =
+  if node.tsNodeNamedChildCount() == 2:
+    var
+      name = getNodeValIf(node.tsNodeNamedChild(1), "type_identifier")
+      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
+      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+      ptrname = false
+
+    if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
+      name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), "type_identifier")
+      ptrname = true
+
+    if name.nBl and name notin gStateRT.types:
+      if ptyp.nBl:
+        # typedef int X
+        gStateRT.types.add(name)
+        ptyp = ptyp.getType()
+        if ptyp != "object" and ptrname:
+          ptyp = &"ptr {ptyp}"
+        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ptyp}        #1 pTypeDefinition()\n"
+      elif ttyp.nBl:
+        # typedef X Y
+        gStateRT.types.add(name)
+        if ptrname:
+          ttyp = &"ptr {ttyp}"
+        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ttyp}        #2 pTypeDefinition()\n"
+      else:
+        case $node.tsNodeNamedChild(0).tsNodeType():
+          of "struct_specifier":
+            pStructSpecifier(node.tsNodeNamedChild(0), name)
+          of "enum_specifier":
+            pEnumSpecifier(node.tsNodeNamedChild(0), name)
+          else:
+            discard
+
+proc pFunctionDeclarator(node: TSNode, typ: string) =
+  if node.tsNodeNamedChildCount() == 2:
+    let
+      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
+
+    if name.nBl and name notin gStateRT.procs and $node.tsNodeNamedChild(1).tsNodeType() == "parameter_list":
+      # typ function(typ param1, ...)
+      var stmt = &"# pFunctionDeclarator()\nproc {name.getIdentifier()}*("
+
+      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
+        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
+          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "parameter_declaration", "identifier", "")
+          if ts.len == 0:
+            return
+          stmt &= ts
+          if i != node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
+            stmt &= ", "
+
+      if typ != "void":
+        stmt &= &"): {typ.getType()} "
+      else:
+        stmt &= ") "
+
+      stmt &= &"{{.importc: \"{name}\", header: {gStateRT.currentHeader}.}}\n"
+
+      gStateRT.procs.add(name)
+      gStateRT.procStr &= stmt
+
+proc pDeclaration*(node: TSNode) =
+  if node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "function_declarator":
+    let
+      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
+      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
+
+    if ptyp.nBl:
+      pFunctionDeclarator(node.tsNodeNamedChild(1), ptyp.getType())
+    elif ttyp.nBl:
+      pFunctionDeclarator(node.tsNodeNamedChild(1), ttyp)
+    elif $node.tsNodeNamedChild(0).tsNodeType() == "struct_specifier" and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
+      let styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
+      if styp.nBl:
+        pFunctionDeclarator(node.tsNodeNamedChild(1), styp)
+
+proc genNimAst(root: TSNode) =
+  var
+    node = root
+    nextnode: TSNode
+
+  while true:
+    if not node.tsNodeIsNull():
+      case $node.tsNodeType():
+        of "ERROR":
+          let (line, col) = getLineCol(node)
+          let file = gStateRT.sourceFile
+          echo &"# [toast] Potentially invalid syntax at {file}:{line}:{col}"
+        of "preproc_def":
+          pPreprocDef(node)
+        of "type_definition":
+          pTypeDefinition(node)
+        of "declaration":
+          pDeclaration(node)
+        of "struct_specifier":
+          if $node.tsNodeParent().tsNodeType() notin ["type_definition", "declaration"]:
+            pStructSpecifier(node)
+        of "enum_specifier":
+          if $node.tsNodeParent.tsNodeType() notin ["type_definition", "declaration"]:
+            pEnumSpecifier(node)
+        else:
+          # TODO: log
+          discard
+    else:
+      return
+
+    if node.tsNodeNamedChildCount() != 0:
+      nextnode = node.tsNodeNamedChild(0)
+    else:
+      nextnode = node.tsNodeNextNamedSibling()
+
+    if nextnode.tsNodeIsNull():
+      while true:
+        node = node.tsNodeParent()
+        if node == root:
+          break
+        if not node.tsNodeNextNamedSibling().tsNodeIsNull():
+          node = node.tsNodeNextNamedSibling()
+          break
+    else:
+      node = nextnode
+
+    if node == root:
+      break
+
+proc printNim*(fullpath: string, root: TSNode) =
+  echo "{.experimental: \"codeReordering\".}"
+
+  var fp = fullpath.replace("\\", "/")
+  gStateRT.currentHeader = getCurrentHeader(fullpath)
+  gStateRT.constStr &= &"  {gStateRT.currentHeader} = \"{fp}\"\n"
+
+  genNimAst(root)
+
+  if gStateRT.constStr.nBl:
+    echo "const\n" & gStateRT.constStr
+
+  if gStateRT.typeStr.nBl:
+    echo "type\n" & gStateRT.typeStr
+
+  if gStateRT.procStr.nBl:
+    echo gStateRT.procStr

nimterop/getters.nim

@@ -21,11 +21,14 @@ proc getLit*(str: string): string =
     str.contains(re"^0x[\d]+$"):
     return str
 
+proc getNodeVal*(node: TSNode): string =
+  return gStateRT.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
+
 proc getNodeValIf*(node: TSNode, esym: string): string =
   if esym != $node.tsNodeType():
     return
 
-  return gStateRT.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
+  return node.getNodeVal()
 
 proc getLineCol*(node: TSNode): tuple[line, col: int] =
   result.line = 1
@@ -81,3 +84,59 @@ proc getPreprocessor*(fullpath: string, mode = "cpp"): string =
               .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";")
           )
   return rdata.join("\n")
+
+converter toString*(kind: Kind): string =
+  return case kind:
+    of exactlyOne:
+      ""
+    of oneOrMore:
+      "+"
+    of zeroOrMore:
+      "*"
+    of zeroOrOne:
+      "?"
+
+converter toKind*(kind: string): Kind =
+  return case kind:
+    of "+":
+      oneOrMore
+    of "*":
+      zeroOrMore
+    of "?":
+      zeroOrOne
+    else:
+      exactlyOne
+
+proc getNameKind*(name: string): tuple[name: string, kind: Kind] =
+  result.name = name
+  result.kind = $name[^1]
+
+  if result.kind != exactlyOne:
+    result.name = name[0 .. ^2]
+
+proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
+  if node.tsNodeNamedChildCount() != 0:
+    for i in 0 .. node.tsNodeNamedChildCount()-1:
+      if $node.tsNodeType() != "comment":
+        result += 1
+
+proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
+  if node.tsNodeNamedChildCount() != 0:
+    for i in 0 .. node.tsNodeNamedChildCount()-1:
+      let
+        name = $node.tsNodeNamedChild(i).tsNodeType()
+
+      if name != "comment":
+        result.add(name)
+
+proc getRegexForAstChildren*(ast: ref Ast): string =
+  result = "^"
+  for i in 0 .. ast.children.len-1:
+    let kind: string = ast.children[i].kind
+    result &= &"(?:{ast.children[i].name}){kind}"
+  result &= "$"
+
+proc getAstChildByName*(ast: ref Ast, name: string): ref Ast =
+  for i in 0 .. ast.children.len-1:
+    if name in ast.children[i].name.split("|"):
+      return ast.children[i]

nimterop/globals.nim

@@ -1,4 +1,18 @@
+import tables
+
 type
+  Kind* = enum
+    exactlyOne
+    oneOrMore     # +
+    zeroOrMore    # *
+    zeroOrOne     # ?
+
+  Ast* = object
+    name*: string
+    kind*: Kind
+    children*: seq[ref Ast]
+    tonim*: proc () {.closure, locks: 0.}
+
   State* = object
     compile*, defines*, headers*, includeDirs*, searchDirs*: seq[string]
 
@@ -9,6 +23,10 @@ type
     code*, constStr*, currentHeader*, mode*, procStr*, typeStr*: string
     sourceFile*: string # eg, C or C++ source or header file
 
+    ast*: Table[string, seq[ref Ast]]
+    data*: seq[tuple[name, val: string]]
+    grammar*: seq[tuple[grammar: string, call: proc() {.locks: 0.}]]
+
 var
   gStateCT* {.compiletime.}: State
   gStateRT*: State
@@ -16,7 +34,6 @@ var
 template nBl*(s: typed): untyped =
   (s.len != 0)
 
-
 type CompileMode* = enum
   c,
   cpp,

nimterop/grammar.nim

@@ -0,0 +1,244 @@
+import strformat, tables
+
+import "."/[getters, globals, lisp]
+
+proc initGrammar() =
+  # #define X Y
+  gStateRT.grammar.add(("""
+   (preproc_def
+    (identifier)
+    (preproc_arg)
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      let
+        name = gStateRT.data[0].val.getIdentifier()
+        val = gStateRT.data[1].val
+
+      if name notin gStateRT.consts:
+        gStateRT.consts.add(name)
+        gStateRT.constStr &= &"  {name}* = {val}\n"
+  ))
+
+  # typedef int X
+  # typedef X Y
+  # typedef struct X Y
+  # typedef ?* Y
+  gStateRT.grammar.add(("""
+   (type_definition
+    (primitive_type|type_identifier?)
+    (struct_specifier?
+     (type_identifier)
+    )
+    (type_identifier?)
+    (pointer_declarator?
+     (type_identifier)
+    )
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      var
+        name = gStateRT.data[1].val.getIdentifier()
+        typ = gStateRT.data[0].val
+
+      if name notin gStateRT.types:
+        gStateRT.types.add(name)
+        gStateRT.typeStr &= &"  {name}* = {typ}\n"
+  ))
+
+  proc pStructCommon(name: string, fstart, fend: int, prefix="") =
+    let
+      nname = name.getIdentifier()
+    if nname notin gStateRT.types:
+      gStateRT.types.add(nname)
+      gStateRT.typeStr &= &"  {nname}* {{.importc: \"{prefix}{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object\n"
+
+      for i in fstart .. gStateRT.data.len-fend:
+        let
+          ftyp = gStateRT.data[i].val
+          fname = gStateRT.data[i+1].val.getIdentifier()
+        gStateRT.typeStr &= &"    {fname}*: {ftyp}\n"
+
+  # struct X {}
+  gStateRT.grammar.add(("""
+   (struct_specifier
+    (type_identifier)
+    (field_declaration_list
+     (field_declaration+
+      (primitive_type|type_identifier?)
+      (struct_specifier?
+       (type_identifier)
+      )
+      (field_identifier?)
+      (pointer_declarator?
+       (field_identifier)
+      )
+     )
+    )
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      pStructCommon(gStateRT.data[0].val, 1, 2, "struct ")
+  ))
+
+  # typedef struct X {}
+  gStateRT.grammar.add(("""
+   (type_definition
+    (struct_specifier
+     (field_declaration_list
+      (field_declaration+
+       (primitive_type|type_identifier?)
+       (struct_specifier?
+        (type_identifier)
+       )
+       (field_identifier?)
+       (pointer_declarator?
+        (field_identifier)
+       )
+      )
+     )
+    )
+    (type_identifier)
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      pStructCommon(gStateRT.data[^1].val, 0, 3)
+  ))
+
+  proc pEnumCommon(name: string, fstart, fend: int, prefix="") =
+    let
+      nname = name.getIdentifier()
+    if nname notin gStateRT.types:
+      gStateRT.types.add(nname)
+      gStateRT.typeStr &= &"  {nname}* = enum\n"
+
+      var
+        i = fstart
+      while i < gStateRT.data.len-fend:
+        let
+          fname = gStateRT.data[i].val.getIdentifier()
+
+        if i+1 < gStateRT.data.len-fend and gStateRT.data[i+1].name == "number_literal":
+          gStateRT.typeStr &= &"    {fname} = {gStateRT.data[i+1].val}\n"
+          i += 2
+        else:
+          gStateRT.typeStr &= &"    {fname}\n"
+          i += 1
+
+  # enum X {}
+  gStateRT.grammar.add(("""
+   (enum_specifier
+    (type_identifier)
+    (enumerator_list
+     (enumerator+
+      (identifier)
+      (number_literal?)
+     )
+    )
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      pEnumCommon(gStateRT.data[0].val, 1, 0)
+  ))
+
+  # typedef enum {} X
+  gStateRT.grammar.add(("""
+   (type_definition
+    (enum_specifier
+     (enumerator_list
+      (enumerator+
+       (identifier)
+       (number_literal?)
+      )
+     )
+    )
+    (type_identifier)
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      pEnumCommon(gStateRT.data[^1].val, 0, 1)
+  ))
+
+  # typ function(typ param1, ...)
+  gStateRT.grammar.add(("""
+   (declaration
+    (primitive_type|type_identifier?)
+    (struct_specifier?
+     (type_identifier)
+    )
+    (function_declarator?
+     (identifier)
+     (parameter_list
+      (parameter_declaration*
+       (primitive_type|type_identifier?)
+       (struct_specifier?
+        (type_identifier)
+       )
+       (enum_specifier?
+        (type_identifier)
+       )
+       (identifier)
+      )
+     )
+    )
+    (pointer_declarator?
+     (function_declarator?
+      (identifier)
+      (parameter_list
+       (parameter_declaration*
+        (primitive_type|type_identifier?)
+        (struct_specifier?
+         (type_identifier)
+        )
+        (enum_specifier?
+         (type_identifier)
+        )
+        (identifier)
+       )
+      )
+     )
+    )
+   )
+  """,
+    proc () {.closure, locks: 0.} =
+      let
+        ftyp = gStateRT.data[0].val
+        fname = gStateRT.data[1].val
+        fnname = fname.getIdentifier()
+
+      if fnname notin gStateRT.procs:
+        var
+          pout = ""
+          i = 2
+        if gStateRT.data.len > 2:
+          while i < gStateRT.data.len-1:
+            let
+              ptyp = gStateRT.data[i].val
+              pname = gStateRT.data[i+1].val.getIdentifier()
+            pout &= &"{pname}: {ptyp},"
+            i += 2
+        if pout.len != 0 and pout[^1] == ',':
+          pout = pout[0 .. ^2]
+
+        gStateRT.procStr &= &"proc {fnname}({pout}): {ftyp} {{.importc: \"{fname}\", header: {gStateRT.currentHeader}.}}\n"
+
+  ))
+
+proc parseGrammar*() =
+  initGrammar()
+
+  gStateRT.ast = initTable[string, seq[ref Ast]]()
+  for i in 0 .. gStateRT.grammar.len-1:
+    var
+      ast = gStateRT.grammar[i].grammar.parseLisp()
+
+    ast.tonim = gStateRT.grammar[i].call
+    if ast.name notin gStateRT.ast:
+      gStateRT.ast[ast.name] = @[ast]
+    else:
+      gStateRT.ast[ast.name].add(ast)
+
+proc printGrammar*() =
+  for name in gStateRT.ast.keys():
+    for ast in gStateRT.ast[name]:
+      echo ast.printAst()

nimterop/lisp.nim

@@ -1,22 +1,18 @@
 import strutils
 import strformat
 
-import globals
+import "."/[getters, globals]
 
 var
-  gTokens {.compiletime.}: seq[string]
+  gTokens: seq[string]
-  idx {.compiletime.} = 0
+  idx = 0
 
-proc tokenize(fullpath: string) =
+proc tokenize(tree: string) =
   var collect = ""
 
   gTokens = @[]
   idx = 0
-  # TODO: consider calling API directly
+  for i in tree:
-  const cmd = &"toast --past --pretty:false --source:{fullpath.quoteShell}"
-  var (output, exitCode) = gorgeEx cmd
-  doAssert exitCode == 0, $exitCode
-  for i in output:
     case i:
       of ' ', '\n', '\r', '(', ')':
         if collect.nBl:
@@ -27,10 +23,6 @@ proc tokenize(fullpath: string) =
       else:
         collect &= $i
 
-  if gTokens.len == 0:
-    echo "toast binary not installed - nimble install nimterop to force build"
-    quit(1)
-
 proc readFromTokens(): ref Ast =
   if idx == gTokens.len:
     echo "Bad AST"
@@ -42,18 +34,12 @@ proc readFromTokens(): ref Ast =
       quit(1)
     if gTokens[idx+1] != "comment":
       result = new(Ast)
-      try:
+      (result.name, result.kind) = gTokens[idx+1].getNameKind()
-        result.sym = parseEnum[Sym](gTokens[idx+1])
-      except:
-        result.sym = IGNORED
-      result.start = gTokens[idx+2].parseInt()
-      result.stop = gTokens[idx+3].parseInt()
       result.children = @[]
-    idx += 4
+    idx += 2
     while gTokens[idx] != ")":
       var res = readFromTokens()
       if not res.isNil():
-        res.parent = result
         result.children.add(res)
   elif gTokens[idx] == ")":
     echo "Poor AST"
@@ -62,7 +48,8 @@ proc readFromTokens(): ref Ast =
   idx += 1
 
 proc printAst*(node: ref Ast, offset=""): string =
-  result = offset & "(" & $node.sym & " " & $node.start & " " & $node.stop
+  result = offset & "(" & node.name & node.kind.toString()
+
   if node.children.len != 0:
     result &= "\n"
     for child in node.children:
@@ -71,7 +58,7 @@ proc printAst*(node: ref Ast, offset=""): string =
   else:
     result &= ")\n"
 
-proc parseLisp*(fullpath: string): ref Ast =
+proc parseLisp*(tree: string): ref Ast =
-  tokenize(fullpath)
+  tokenize(tree)
 
   return readFromTokens()