Support type qualifiers

config.nims

@@ -9,4 +9,12 @@ when defined(Windows):
   switch("gc", "markAndSweep")
 
 # Retain stackTrace for clear errors
-switch("stackTrace", "on")
+switch("stackTrace", "on")
+
+# Path to compiler
+switch("path", "$nim")
+
+# Case objects
+when not defined(danger):
+  switch("define", "nimOldCaseObjects")
+

nimterop.nimble

@@ -25,7 +25,7 @@ proc execTest(test: string) =
   execCmd "nim cpp -r " & test
 
 task buildToast, "build toast":
-  execCmd("nim c -f -d:danger nimterop/toast.nim")
+  execCmd("nim c -f nimterop/toast.nim")
 
 task bt, "build toast":
   execCmd("nim c -d:danger nimterop/toast.nim")
@@ -36,19 +36,22 @@ task docs, "Generate docs":
 task test, "Test":
   buildToastTask()
 
-  execTest "tests/tnimterop_c.nim"
-  execCmd "nim cpp -f -r tests/tnimterop_cpp.nim"
-  execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h"
-  execTest "tests/tpcre.nim"
+  execTest "tests/tnimterop2.nim"
+
+  # Commented out until newalgo is ready
+  # execTest "tests/tnimterop_c.nim"
+  # execCmd "nim cpp -f -r tests/tnimterop_cpp.nim"
+  # execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h"
+  # execTest "tests/tpcre.nim"
 
   # Platform specific tests
-  when defined(Windows):
-    execTest "tests/tmath.nim"
-  if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"):
-    execTest "tests/tsoloud.nim"
+  # when defined(Windows):
+  #   execTest "tests/tmath.nim"
+  # if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"):
+  #   execTest "tests/tsoloud.nim"
 
   # getHeader tests
-  withDir("tests"):
-    execCmd("nim e getheader.nims")
+  # withDir("tests"):
+  #   execCmd("nim e getheader.nims")
 
   docsTask()

nimterop/all.nim

@@ -4,4 +4,4 @@ Module that should import everything so that `nim doc --project nimtero/all` run
 
 # TODO: make sure it does import everything.
 
-import "."/[cimport, build, types, plugin, compat]
+import "."/[cimport, build, types, plugin]

nimterop/ast2.nim

@@ -0,0 +1,857 @@
+import macros, os, sets, strutils, tables, times
+
+import compiler/[ast, idents, options, renderer]
+
+import "."/treesitter/api
+
+import "."/[globals, getters]
+
+proc addConst(nimState: NimState, node: TSNode) =
+  # #define X Y
+  #
+  # (preproc_def
+  #  (identifier)
+  #  (preproc_arg)
+  # )
+  decho("addConst()")
+  nimState.printDebug(node)
+
+  if node[0].getName() == "identifier" and
+    node[1].getName() == "preproc_arg":
+    let
+      constDef = newNode(nkConstDef)
+
+      # node[0] = identifier = const name
+      (name, info) = nimState.getNameInfo(node.getAtom(), nskConst)
+      # TODO - check blank and override
+      ident = nimState.getIdent(name, info)
+
+      # node[1] = preproc_arg = value
+      val = nimState.getNodeVal(node[1]).getLit()
+
+    # If supported literal
+    if val.kind != nkNilLit:
+      # const X* = Y
+      #
+      # nkConstDef(
+      #  nkPostfix(
+      #   nkIdent("*"),
+      #   nkIdent("X")
+      #  ),
+      #  nkEmpty(),
+      #  nkXLit(Y)
+      # )
+      constDef.add ident
+      constDef.add newNode(nkEmpty)
+      constDef.add val
+
+      # nkConstSection.add
+      nimState.constSection.add constDef
+
+      nimState.printDebug(constDef)
+
+proc newTypeIdent(nimState: NimState, node: TSNode, override = ""): PNode =
+  # Create nkTypeDef PNode with first ident
+  #
+  # If `override`, use it instead of node.getAtom() for name
+  let
+    (name, info) = nimState.getNameInfo(node.getAtom(), nskType)
+    # TODO - check blank and override
+    ident =
+      if override.len != 0:
+        nimState.getIdent(override, info)
+      else:
+        nimState.getIdent(name, info)
+
+  # type name* =
+  #
+  # nkTypeDef(
+  #  nkPostfix(
+  #   nkIdent("*"),
+  #   nkIdent(name)
+  #  ),
+  #  nkEmpty()
+  # )
+  result = newNode(nkTypeDef)
+  result.add ident
+  result.add newNode(nkEmpty)
+
+proc newPtrTree(nimState: NimState, count: int, typ: PNode): PNode =
+  # Create nkPtrTy tree depending on count
+  #
+  # Reduce by 1 if Nim type available for ptr X - e.g. ptr cchar = cstring
+  var
+    count = count
+    chng = false
+  if typ.kind == nkIdent:
+    let
+      tname = typ.ident.s
+      ptname = getPtrType(tname)
+    if tname != ptname:
+      # If Nim type available, use that ident
+      result = nimState.getIdent(ptname, typ.info, exported = false)
+      chng = true
+      # One ptr reduced
+      count -= 1
+  if count > 0:
+    # Nested nkPtrTy(typ) depending on count
+    #
+    # [ptr ...] typ
+    #
+    # nkPtrTy(
+    #  nkPtrTy(
+    #    typ
+    #  )
+    # )
+    result = newNode(nkPtrTy)
+    var
+      parent = result
+      child: PNode
+    for i in 1 ..< count:
+      child = newNode(nkPtrTy)
+      parent.add child
+      parent = child
+    parent.add typ
+  elif not chng:
+    # Either no ptr, or none left after Nim type adjustment
+    result = typ
+
+proc newArrayTree(nimState: NimState, node: TSNode, typ, size: PNode): PNode =
+  # Create nkBracketExpr tree depending on input
+  let
+    (_, info) = nimState.getNameInfo(node, nskType)
+    ident = nimState.getIdent("array", info, exported = false)
+
+  # array[size, typ]
+  #
+  # nkBracketExpr(
+  #  nkIdent("array"),
+  #  size,
+  #  typ
+  # )
+  result = newNode(nkBracketExpr)
+  result.add ident
+  result.add size
+  result.add typ
+
+proc getTypeArray(nimState: NimState, node: TSNode): PNode
+proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode
+
+proc newIdentDefs(nimState: NimState, name: string, node: TSNode, offset: SomeInteger, exported = false): PNode =
+  # Create nkIdentDefs tree for specified proc parameter or object field
+  #
+  # For proc, param should not be exported
+  #
+  # pname: [ptr ..] typ
+  #
+  # nkIdentDefs(
+  #  nkIdent(pname),
+  #  typ,
+  #  nkEmpty()
+  # )
+  #
+  # For object, field should be exported
+  #
+  # pname*: [ptr ..] typ
+  #
+  # nkIdentDefs(
+  #  nkPostfix(
+  #   nkIdent("*"),
+  #   nkIdent(pname)
+  #  ),
+  #  typ,
+  #  nkEmpty()
+  # )
+  result = newNode(nkIdentDefs)
+
+  let
+    start = getStartAtom(node)
+
+    # node[start] - param type
+    (tname, tinfo) = nimState.getNameInfo(node[start], nskType)
+    tident = nimState.getIdent(tname, tinfo, exported = false)
+
+  if start == node.len - 1:
+    # Only for proc with no named param - create a param name based on offset
+    #
+    # int func(char, int);
+    let
+      pname = "a" & $(offset+1)
+      pident = nimState.getIdent(pname, tinfo, exported)
+    result.add pident
+    result.add tident
+    result.add newNode(nkEmpty)
+  else:
+    let
+      fdecl = node[start+1].anyChildInTree("function_declarator")
+      adecl = node[start+1].anyChildInTree("array_declarator")
+      abst = node[start+1].getName() == "abstract_pointer_declarator"
+    if fdecl.isNil and adecl.isNil:
+      if abst:
+        # Only for proc with no named param with pointer type
+        # Create a param name based on offset
+        #
+        # int func(char *, int **);
+        let
+          pname = "a" & $(offset+1)
+          pident = nimState.getIdent(pname, tinfo, exported)
+          acount = node[start+1].getXCount("abstract_pointer_declarator")
+        result.add pident
+        result.add nimState.newPtrTree(acount, tident)
+        result.add newNode(nkEmpty)
+      else:
+        # Named param, simple type
+        let
+          (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name)
+          pident = nimState.getIdent(pname, pinfo, exported)
+
+          count = node[start+1].getPtrCount()
+        result.add pident
+        if count > 0:
+          result.add nimState.newPtrTree(count, tident)
+        else:
+          result.add tident
+        result.add newNode(nkEmpty)
+    elif not fdecl.isNil:
+      # Named param, function pointer
+      let
+        (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name)
+        pident = nimState.getIdent(pname, pinfo, exported)
+      result.add pident
+      result.add nimState.getTypeProc(name, node)
+      result.add newNode(nkEmpty)
+    elif not adecl.isNil:
+      # Named param, array type
+      let
+        (pname, pinfo) = nimState.getNameInfo(node[start+1].getAtom(), nskField, parent = name)
+        pident = nimState.getIdent(pname, pinfo, exported)
+      result.add pident
+      result.add nimState.getTypeArray(node)
+      result.add newNode(nkEmpty)
+    else:
+      result = nil
+
+proc newProcTree(nimState: NimState, name: string, node: TSNode, rtyp: PNode): PNode =
+  # Create nkProcTy tree for specified proc type
+  let
+    fparam = newNode(nkFormalParams)
+
+  # Add return type
+  fparam.add rtyp
+
+  if not node.isNil:
+    for i in 0 ..< node.len:
+      # Add nkIdentDefs for each param
+      let
+        param = nimState.newIdentDefs(name, node[i], i, exported = false)
+      if not param.isNil:
+        fparam.add param
+
+  # proc(pname: ptyp ..): rtyp
+  #
+  # nkProcTy(
+  #  nkFormalParams(
+  #   rtyp,
+  #   nkIdentDefs(     # multiple depending on params
+  #    ..
+  #   )
+  #  ),
+  #  nkEmpty()
+  # )
+  result = newNode(nkProcTy)
+  result.add fparam
+  result.add newNode(nkEmpty)
+
+proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode =
+  # Create nkRecList tree for specified object
+  if not node.isNil:
+    # fname*: ftyp
+    # ..
+    #
+    # nkRecList(
+    #  nkIdentDefs(      # multiple depending on fields
+    #   ..
+    #  )
+    # )
+    result = newNode(nkRecList)
+
+    for i in 0 ..< node.len:
+      # Add nkIdentDefs for each field
+      let
+        field = nimState.newIdentDefs(name, node[i], i, exported = true)
+      if not field.isNil:
+        result.add field
+
+proc addTypeObject(nimState: NimState, node: TSNode, override = "", duplicate = "") =
+  # Add a type of object
+  #
+  # If `override` is set, use it as the name
+  # If `duplicate` is set, don't add the same name
+  decho("addTypeObject()")
+  let
+    # TODO - check blank and override
+    typeDef = nimState.newTypeIdent(node, override)
+    name = $typeDef[0][1]
+
+  if name != duplicate:
+    # type X* = object
+    #
+    # nkTypeDef(
+    #  nkPostfix(
+    #   nkIdent("*"),
+    #   nkIdent("X")
+    #  ),
+    #  nkEmpty(),
+    #  nkObjectTy(
+    #   nkEmpty(),
+    #   nkEmpty(),
+    #   nkEmpty()
+    #  )
+    # )
+    let
+      obj = newNode(nkObjectTy)
+    obj.add newNode(nkEmpty)
+    obj.add newNode(nkEmpty)
+
+    let
+      fdlist = node.anyChildInTree("field_declaration_list")
+    if not fdlist.isNil and fdlist.len > 0:
+      # Add fields to object if present
+      obj.add nimState.newRecListTree(name, fdlist)
+    else:
+      obj.add newNode(nkEmpty)
+
+    typeDef.add obj
+
+    # nkTypeSection.add
+    nimState.typeSection.add typeDef
+
+    nimState.printDebug(typeDef)
+
+proc addTypeTyped(nimState: NimState, node: TSNode, toverride = "", duplicate = "") =
+  # Add a type of a specified type
+  #
+  # If `toverride` is set, use it as the type name
+  # If `duplicate` is set, don't add the same name
+  decho("addTypeTyped()")
+  let
+    start = getStartAtom(node)
+  for i in start+1 ..< node.len:
+    # Add a type of a specific type
+    let
+      # node[i] = identifer = name
+      # TODO - check blank and override
+      typeDef = nimState.newTypeIdent(node[i])
+
+      # node[start] = identifier = type name
+      (tname0, tinfo) = nimState.getNameInfo(node[start].getAtom(), nskType)
+
+      # Override type name
+      tname = if toverride.len != 0: toverride else: tname0
+
+      ident = nimState.getIdent(tname, tinfo, exported = false)
+
+      # node[i] could have nested pointers
+      count = node[i].getPtrCount()
+
+    # Skip typedef X X;
+    if $typeDef[0][1] != tname:
+      if count > 0:
+        # If pointers
+        typeDef.add nimState.newPtrTree(count, ident)
+      else:
+        typeDef.add ident
+
+      # type X* = [ptr ..] Y
+      #
+      # nkTypeDef(
+      #  nkPostfix(
+      #   nkIdent("*"),
+      #   nkIdent("X")
+      #  ),
+      #  nkEmpty(),
+      #  nkPtrTy(            # optional, nested
+      #   nkIdent("Y")
+      #  )
+      # )
+
+      # nkTypeSection.add
+      nimState.typeSection.add typeDef
+
+      nimState.printDebug(typeDef)
+    else:
+      nimState.addTypeObject(node, duplicate = duplicate)
+
+proc getTypeArray(nimState: NimState, node: TSNode): PNode =
+  # Create array type tree
+  let
+    start = getStartAtom(node)
+
+    # node[start] = identifier = type name
+    (name, info) = nimState.getNameInfo(node[start].getAtom(), nskType)
+    ident = nimState.getIdent(name, info, exported = false)
+
+    # Top-most array declarator
+    adecl = node[start+1].firstChildInTree("array_declarator")
+
+    # node[start+1] could have nested arrays
+    acount = adecl.getArrayCount()
+    innermost = adecl.mostNestedChildInTree()
+
+    # node[start+1] could have nested pointers - type
+    tcount = node[start+1].getPtrCount()
+
+    # Name could be nested pointer to array
+    #
+    # (..
+    #  (array_declarator
+    #   (parenthesized_declarator
+    #    (pointer_declarator ..
+    #     (pointer_declarator          <- search upwards from atom
+    #      (type_identifier)           <- atom
+    #     )
+    #    )
+    #   )
+    #  )
+    # )
+    ncount = innermost[0].getAtom().tsNodeParent().getPtrCount(reverse = true)
+
+  result = ident
+  var
+    cnode = adecl
+
+  if tcount > 0:
+    # If pointers
+    result = nimState.newPtrTree(tcount, result)
+
+  for i in 0 ..< acount:
+    let
+      size = nimState.getNodeVal(cnode[1]).getLit()
+    if size.kind != nkNilLit:
+      result = nimState.newArrayTree(cnode, result, size)
+      cnode = cnode[0]
+
+  if ncount > 0:
+    result = nimState.newPtrTree(ncount, result)
+
+proc addTypeArray(nimState: NimState, node: TSNode) =
+  # Add a type of array type
+  decho("addTypeArray()")
+  let
+    # node[1] = identifer = name
+    # TODO - check blank and override
+    typeDef = nimState.newTypeIdent(node[1])
+
+    typ = nimState.getTypeArray(node)
+
+  typeDef.add typ
+
+  # type X* = [ptr] array[x, [ptr] Y]
+  #
+  # nkTypeDef(
+  #  nkPostfix(
+  #   nkIdent("*"),
+  #   nkIdent("X")
+  #  ),
+  #  nkEmpty(),
+  #  nkPtrTy(              # optional, nested
+  #   nkBracketExpr(
+  #    nkIdent("array")
+  #    nkXLit(x),
+  #    nkPtrTy(            # optional, nested
+  #     nkIdent("Y")
+  #    )
+  #   )
+  #  )
+  # )
+
+  # nkTypeSection.add
+  nimState.typeSection.add typeDef
+
+  nimState.printDebug(typeDef)
+
+proc getTypeProc(nimState: NimState, name: string, node: TSNode): PNode =
+  # Create proc type tree
+  let
+    # node[0] = identifier = return type name
+    (rname, rinfo) = nimState.getNameInfo(node[0].getAtom(), nskType)
+
+    # Parameter list
+    plist = node[1].anyChildInTree("parameter_list")
+
+    # node[1] could have nested pointers
+    tcount = node[1].getPtrCount()
+
+    # Name could be nested pointer to function
+    #
+    # (..
+    #  (function_declarator
+    #   (parenthesized_declarator
+    #    (pointer_declarator ..
+    #     (pointer_declarator          <- search upwards from atom
+    #      (type_identifier)           <- atom
+    #     )
+    #    )
+    #   )
+    #  )
+    # )
+    ncount = node[1].getAtom().tsNodeParent().getPtrCount(reverse = true)
+
+  # Return type
+  var
+    retType = nimState.getIdent(rname, rinfo, exported = false)
+  if tcount > 0:
+    retType = nimState.newPtrTree(tcount, retType)
+
+  # Proc with return type and params
+  result = nimState.newProcTree(name, plist, retType)
+  if ncount > 1:
+    result = nimState.newPtrTree(ncount-1, result)
+
+proc addTypeProc(nimState: NimState, node: TSNode) =
+  # Add a type of proc type
+  decho("addTypeProc()")
+  let
+    # node[1] = identifier = name
+    # TODO - check blank and override
+    typeDef = nimState.newTypeIdent(node[1])
+    name = $typeDef[0][1]
+
+    procTy = nimState.getTypeProc(name, node)
+
+  typeDef.add procTy
+
+  # type X* = proc(a1: Y, a2: Z): P
+  #
+  # nkTypeDef(
+  #  nkPostfix(
+  #   nkIdent("*"),
+  #   nkIdent("X")
+  #  ),
+  #  nkEmpty(),
+  #  nkPtrTy(              # optional, nested
+  #   nkProcTy(
+  #    nkFormalParams(
+  #     nkPtrTy(           # optional, nested
+  #      nkIdent(retType)
+  #     ),
+  #     nkIdentDefs(
+  #      nkIdent(param),
+  #      nkPtrTy(
+  #       nkIdent(ptype)
+  #      ),
+  #      nkEmpty()
+  #     ),
+  #     ...
+  #    ),
+  #    nkEmpty()
+  #   )
+  #  )
+  # )
+
+  # nkTypeSection.add
+  nimState.typeSection.add typeDef
+
+  nimState.printDebug(typeDef)
+
+proc addType(nimState: NimState, node: TSNode) =
+  decho("addType()")
+  nimState.printDebug(node)
+
+  if node.getName() == "struct_specifier":
+    # struct X;
+    #
+    # (struct_specifier
+    #  (type_identifier)
+    # )
+    #
+    # struct X {};
+    #
+    # (struct_specifier
+    #  (type_identifier)
+    #  (field_declaration_list = "{}")
+    # )
+    #
+    # struct X { char *a1; };
+    #
+    # (struct_specifier
+    #  (type_identifier)
+    #  (field_declaration_list
+    #   (field_declaration
+    #    (type_identifier|primitive_type|)
+    #    (struct_specifier
+    #     (type_identifier)
+    #    )
+    #
+    #    (field_identifier)
+    #   )
+    #   (field_declaration ...)
+    #  )
+    decho("addType(): case 1")
+    nimState.addTypeObject(node)
+  elif node.getName() == "type_definition":
+    if node.len >= 2:
+      let
+        fdlist = node[0].anyChildInTree("field_declaration_list")
+      if (fdlist.isNil or (not fdlist.isNil and fdlist.len == 0)) and
+          nimState.getNodeVal(node[1]) == "":
+        # typedef struct X;
+        #
+        # (type_definition
+        #  (struct_specifier
+        #   (type_identifier)
+        #  )
+        #  (type_definition = "")
+        # )
+        #
+        # typedef struct X {};
+        #
+        # (type_definition
+        #  (struct_specifier
+        #   (type_identifier)
+        #   (field_declaration_list = "{}")
+        #  )
+        #  (type_definition = "")
+        # )
+        decho("addType(): case 2")
+        nimState.addTypeObject(node[0])
+      else:
+        let
+          fdecl = node[1].anyChildInTree("function_declarator")
+          adecl = node[1].anyChildInTree("array_declarator")
+        if fdlist.isNil():
+          if adecl.isNil and fdecl.isNil:
+            # typedef X Y;
+            # typedef X *Y;
+            # typedef struct X Y;
+            # typedef struct X *Y;
+            #
+            # (type_definition
+            #  (type_qualifier?)
+            #  (type_identifier|primitive_type|)
+            #  (struct_specifier
+            #   (type_identifier)
+            #  )
+            #
+            #  (pointer_declarator - optional, nested
+            #   (type_identifier)
+            #  )
+            # )
+            decho("addType(): case 3")
+            nimState.addTypeTyped(node)
+          elif not fdecl.isNil:
+            # typedef X (*Y)(a1, a2, a3);
+            # typedef X *(*Y)(a1, a2, a3);
+            # typedef struct X (*Y)(a1, a2, a3);
+            # typedef struct X *(*Y)(a1, a2, a3);
+            #
+            # (type_definition
+            #  (type_qualifier?)
+            #  (type_identifier|primitive_type|)
+            #  (struct_specifier
+            #   (type_identifier)
+            #  )
+            #
+            #  (pointer_declarator - optional, nested
+            #   (function_declarator
+            #    (parenthesized_declarator
+            #     (pointer_declarator
+            #      (type_identifer)
+            #     )
+            #    )
+            #    (parameter_list
+            #     (parameter_declaration
+            #      (struct_specifier|type_identifier|primitive_type|array_declarator|function_declarator)
+            #      (identifier - optional)
+            #     )
+            #    )
+            #   )
+            #  )
+            # )
+            decho("addType(): case 4")
+            nimState.addTypeProc(node)
+          elif not adecl.isNil:
+            # typedef struct X Y[a][..];
+            # typedef struct X *Y[a][..];
+            # typedef struct X *(*Y)[a][..];
+            #
+            # (type_definition
+            #  (type_qualifier?)
+            #  (type_identifier|primitive_type|)
+            #  (struct_specifier
+            #   (type_identifier)
+            #  )
+            #
+            #  (pointer_declarator - optional, nested
+            #   (array_declarator - nested
+            #    (pointer_declarator - optional, nested
+            #     (type_identifier)
+            #    )
+            #    (number_literal)
+            #   )
+            #  )
+            # )
+            decho("addType(): case 5")
+            nimState.addTypeArray(node)
+        else:
+          if node.firstChildInTree("field_declaration_list").isNil:
+            # typedef struct X { .. } Y, *Z;
+            #
+            # (type_definition
+            #  (struct_specifier
+            #   (type_identifier) - named struct          <====
+            #   (field_declaration_list
+            #    (field_declaration - optional, multiple
+            #     (type_identifier|primitive_type|)
+            #     (function_declarator|array_declarator
+            #      ..
+            #     )
+            #
+            #     (field_identifier)
+            #    )
+            #   )
+            #  )
+            #
+            #  (type_identifier)
+            #  (pointer_declarator - optional, nested
+            #   (type_identifier)
+            #  )
+            # )
+
+            # First add struct as object
+            decho("addType(): case 6")
+            nimState.addTypeObject(node[0])
+
+            if node.len > 1 and nimState.getNodeVal(node[1]) != "":
+              # Add any additional names
+              nimState.addTypeTyped(node, duplicate = nimState.getNodeVal(node[0].getAtom()))
+          else:
+            # Same as above except unnamed struct
+            #
+            # typedef struct { .. } Y, *Z;
+
+            # Get any name that isn't a pointer
+            decho("addType(): case 7")
+            let
+              name = block:
+                var
+                  name = ""
+                for i in 1 ..< node.len:
+                  if node[i].getName() == "type_identifier":
+                    name = nimState.getNodeVal(node[i].getAtom())
+
+                name
+
+            # Now add struct as object with specified name
+            nimState.addTypeObject(node[0], override = name)
+
+            if name.len != 0:
+              # Add any additional names except duplicate
+              nimState.addTypeTyped(node, toverride = name, duplicate = name)
+
+proc addEnum(nimState: NimState, node: TSNode) =
+  decho("addEnum()")
+  nimState.printDebug(node)
+
+proc addProc(nimState: NimState, node: TSNode) =
+  decho("addProc()")
+  nimState.printDebug(node)
+
+proc processNode(nimState: NimState, node: TSNode): bool =
+  result = true
+
+  case node.getName()
+  of "preproc_def":
+    nimState.addConst(node)
+  of "type_definition":
+    if not node.firstChildInTree("enum_specifier").isNil():
+      nimState.addEnum(node)
+    else:
+      nimState.addType(node)
+  of "struct_specifier":
+    nimState.addType(node)
+  of "enum_specifier":
+    nimState.addEnum(node)
+  of "declaration":
+    nimState.addProc(node)
+  else:
+    # Unknown
+    result = false
+
+proc searchTree(nimState: NimState, root: TSNode) =
+  # Search AST generated by tree-sitter for recognized elements
+  var
+    node = root
+    nextnode: TSNode
+    depth = 0
+    processed = false
+
+  while true:
+    if not node.isNil() and depth > -1:
+      processed = nimState.processNode(node)
+    else:
+      break
+
+    if not processed and node.len() != 0:
+      nextnode = node[0]
+      depth += 1
+    else:
+      nextnode = node.tsNodeNextNamedSibling()
+
+    if nextnode.isNil():
+      while true:
+        node = node.tsNodeParent()
+        depth -= 1
+        if depth == -1:
+          break
+        if node == root:
+          break
+        if not node.tsNodeNextNamedSibling().isNil():
+          node = node.tsNodeNextNamedSibling()
+          break
+    else:
+      node = nextnode
+
+    if node == root:
+      break
+
+proc printNimHeader*(gState: State) =
+  gecho """# Generated at $1
+# Command line:
+#   $2 $3
+
+{.hint[ConvFromXtoItselfNotNeeded]: off.}
+
+import nimterop/types
+""" % [$now(), getAppFilename(), commandLineParams().join(" ")]
+
+proc printNim*(gState: State, fullpath: string, root: TSNode) =
+  var
+    nimState = new(NimState)
+    #fp = fullpath.replace("\\", "/")
+
+  nimState.identifiers = newTable[string, string]()
+
+  nimState.gState = gState
+  nimState.currentHeader = getCurrentHeader(fullpath)
+  nimState.sourceFile = fullpath
+
+  # Nim compiler objects
+  nimState.identCache = newIdentCache()
+  nimState.config = newConfigRef()
+
+  nimState.constSection = newNode(nkConstSection)
+  nimState.enumSection = newNode(nkStmtList)
+  nimState.procSection = newNode(nkStmtList)
+  nimState.typeSection = newNode(nkTypeSection)
+
+  nimState.searchTree(root)
+
+  var
+    tree = newNode(nkStmtList)
+  tree.add nimState.enumSection
+  tree.add nimState.constSection
+  tree.add nimState.typeSection
+  tree.add nimState.procSection
+
+  gecho tree.renderTree()

nimterop/build.nim

@@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables
 
 import os except findExe, sleep
 
-import "."/[compat]
-
 proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string =
   result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)])
   if not noQuote:
@@ -648,7 +646,7 @@ proc getGccPaths*(mode = "c"): seq[string] =
       break
     if inc:
       var
-        path = line.strip().myNormalizedPath()
+        path = line.strip().normalizedPath()
       if path notin result:
         result.add path
 
@@ -667,13 +665,13 @@ proc getGccLibPaths*(mode = "c"): seq[string] =
     if "LIBRARY_PATH=" in line:
       for path in line[13 .. ^1].split(PathSep):
         var
-          path = path.strip().myNormalizedPath()
+          path = path.strip().normalizedPath()
         if path notin result:
           result.add path
       break
     elif '\t' in line:
       var
-        path = line.strip().myNormalizedPath()
+        path = line.strip().normalizedPath()
       if path notin result:
         result.add path
 

nimterop/cimport.nim

@@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1
 
 include "."/globals
 
-import "."/[build, compat, paths, types]
+import "."/[build, paths, types]
 export types
 
 proc interpPath(dir: string): string=

nimterop/docs.nim

@@ -1,22 +1,18 @@
 import macros, strformat
 
-when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
-  from os import parentDir, getCurrentCompilerExe, DirSep
-  proc getNimRootDir(): string =
-    #[
-    hack, but works
-    alternatively (but more complex), use (from a nim file, not nims otherwise
-    you get Error: ambiguous call; both system.fileExists):
-    import "$nim/testament/lib/stdtest/specialpaths.nim"
-    nimRootDir
-    ]#
-    fmt"{currentSourcePath}".parentDir.parentDir.parentDir
-else:
-  proc getCurrentCompilerExe*(): string =
-    "nim"
+from os import parentDir, getCurrentCompilerExe, DirSep
+proc getNimRootDir(): string =
+  #[
+  hack, but works
+  alternatively (but more complex), use (from a nim file, not nims otherwise
+  you get Error: ambiguous call; both system.fileExists):
+  import "$nim/testament/lib/stdtest/specialpaths.nim"
+  nimRootDir
+  ]#
+  fmt"{currentSourcePath}".parentDir.parentDir.parentDir
 
-  const
-    DirSep = when defined(windows): '\\' else: '/'
+const
+  DirSep = when defined(windows): '\\' else: '/'
 
 proc execAction(cmd: string): string =
   var
@@ -80,4 +76,4 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath
     for i in 0 .. paramCount():
       if paramStr(i) == "--publish":
         echo execAction(&"cd {path} && ghp-import --no-jekyll -fp {path}")
-        break
+        break

nimterop/getters.nim

@@ -2,7 +2,9 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times
 
 import regex
 
-import "."/[build, compat, globals, plugin, treesitter/api]
+import compiler/[ast, idents, lineinfos, msgs, pathutils, renderer]
+
+import "."/[build, globals, plugin, treesitter/api]
 
 const gReserved = """
 addr and as asm
@@ -26,6 +28,10 @@ when while
 xor
 yield""".split(Whitespace).toHashSet()
 
+# Types related
+
+# Types related
+
 const gTypeMap* = {
   # char
   "char": "cchar",
@@ -91,6 +97,17 @@ proc getType*(str: string): string =
   if gTypeMap.hasKey(result):
     result = gTypeMap[result]
 
+proc getPtrType*(str: string): string =
+  result = case str:
+    of "cchar":
+      "cstring"
+    of "object":
+      "pointer"
+    else:
+      str
+
+# Identifier related
+
 proc checkIdentifier(name, kind, parent, origName: string) =
   let
     parentStr = if parent.nBl: parent & ":" else: ""
@@ -165,6 +182,8 @@ proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool
       nimState.identifiers[nimName] = name
       result = true
 
+# Overrides related
+
 proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string =
   doAssert name.nBl, "Blank identifier error"
 
@@ -189,34 +208,146 @@ proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string =
     for i in nimState.gState.onSymbolOverrideFinal(typ):
       result &= "\n" & nimState.getOverride(i, kind)
 
-proc getPtrType*(str: string): string =
-  result = case str:
-    of "ptr cchar":
-      "cstring"
-    of "ptr ptr cchar":
-      "ptr cstring"
-    of "ptr object":
-      "pointer"
-    of "ptr ptr object":
-      "ptr pointer"
-    else:
-      str
-
-proc getLit*(str: string): string =
+proc getLit*(str: string): PNode =
   # Used to convert #define literals into const
   let
     str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip()
 
-  if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal
-    str.contains(re"^0x[\da-fA-F]+$") or        # hexadecimal
-    str.contains(re"^'[[:ascii:]]'$") or        # char
-    str.contains(re"""^"[[:ascii:]]+"$"""):     # char *
-    return str
+  if str.contains(re"^[\-]?[\d]+$"):              # decimal
+    result = newIntNode(nkIntLit, parseInt(str))
+
+  elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"):   # float
+    result = newFloatNode(nkFloatLit, parseFloat(str))
+
+  # TODO - hex becomes int on render
+  elif str.contains(re"^0x[\da-fA-F]+$"):         # hexadecimal
+    result = newIntNode(nkIntLit, parseHexInt(str))
+
+  elif str.contains(re"^'[[:ascii:]]'$"):         # char
+    result = newNode(nkCharLit)
+    result.intVal = str[1].int64
+
+  elif str.contains(re"""^"[[:ascii:]]+"$"""):    # char *
+    result = newStrNode(nkStrLit, str[1 .. ^2])
+
+  else:
+    result = newNode(nkNilLit)
+
+# TSNode shortcuts
+
+proc isNil*(node: TSNode): bool =
+  node.tsNodeIsNull()
+
+proc len*(node: TSNode): int =
+  if not node.isNil:
+    result = node.tsNodeNamedChildCount().int
+
+proc `[]`*(node: TSNode, i: SomeInteger): TSNode =
+  if i < node.len():
+    result = node.tsNodeNamedChild(i.uint32)
+
+proc getName*(node: TSNode): string {.inline.} =
+  if not node.isNil:
+    return $node.tsNodeType()
+
+proc getNodeVal*(gState: State, node: TSNode): string =
+  if not node.isNil:
+    return gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
 
 proc getNodeVal*(nimState: NimState, node: TSNode): string =
-  return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
+  nimState.gState.getNodeVal(node)
+
+proc getAtom*(node: TSNode): TSNode =
+  if not node.isNil:
+    # Get child node which is topmost atom
+    if node.getName() in gAtoms:
+      return node
+    elif node.len() != 0:
+      return node[0].getAtom()
+
+proc getStartAtom*(node: TSNode): int =
+  if not node.isNil:
+    # Skip const, volatile and other type qualifiers
+    for i in 0 .. node.len - 1:
+      if node[i].getAtom().getName() notin gAtoms:
+        result += 1
+      else:
+        break
+
+proc getXCount*(node: TSNode, ntype: string, reverse = false): int =
+  if not node.isNil:
+    # Get number of ntype nodes nested in tree
+    var
+      cnode = node
+    while ntype in cnode.getName():
+      result += 1
+      if reverse:
+        cnode = cnode.tsNodeParent()
+      else:
+        if cnode.len() != 0:
+          cnode = cnode[0]
+        else:
+          break
+
+proc getPtrCount*(node: TSNode, reverse = false): int =
+  node.getXCount("pointer_declarator")
+
+proc getArrayCount*(node: TSNode, reverse = false): int =
+  node.getXCount("array_declarator")
+
+proc getDeclarator*(node: TSNode): TSNode =
+  if not node.isNil:
+    # Return if child is a function or array declarator
+    if node.getName() in ["function_declarator", "array_declarator"]:
+      return node
+    elif node.len() != 0:
+      return node[0].getDeclarator()
+
+proc firstChildInTree*(node: TSNode, ntype: string): TSNode =
+  # Search for node type in tree - first children
+  var
+    cnode = node
+  while not cnode.isNil:
+    if cnode.getName() == ntype:
+      return cnode
+    cnode = cnode[0]
+
+proc anyChildInTree*(node: TSNode, ntype: string): TSNode =
+  # Search for node type anywhere in tree - depth first
+  var
+    cnode = node
+  while not cnode.isNil:
+    if cnode.getName() == ntype:
+      return cnode
+    for i in 0 ..< cnode.len:
+      let
+        ccnode = cnode[i].anyChildInTree(ntype)
+      if not ccnode.isNil():
+        return ccnode
+    if cnode != node:
+      cnode = cnode.tsNodeNextNamedSibling()
+    else:
+      break
+
+proc mostNestedChildInTree*(node: TSNode): TSNode =
+  # Search for the most nested child of node's type in tree
+  var
+    cnode = node
+    ntype = cnode.getName()
+  while not cnode.isNil and cnode.len != 0 and cnode[0].getName() == ntype:
+    cnode = cnode[0]
+  result = cnode
+
+proc inChildren*(node: TSNode, ntype: string): bool =
+  # Search for node type in immediate children
+  result = false
+  for i in 0 ..< node.len():
+    if (node[i]).getName() == ntype:
+      result = true
+      break
 
 proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
+  # Get line number and column info for node
   result.line = 1
   result.col = 1
   for i in 0 .. node.tsNodeStartByte().int-1:
@@ -225,6 +356,150 @@ proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
       result.line += 1
     result.col += 1
 
+proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
+  for i in 0 ..< node.len():
+    if node.getName() != "comment":
+      result += 1
+
+proc getPxName*(node: TSNode, offset: int): string =
+  # Get the xth (grand)parent of the node
+  var
+    np = node
+    count = 0
+
+  while not np.isNil() and count < offset:
+    np = np.tsNodeParent()
+    count += 1
+
+  if count == offset and not np.isNil():
+    return np.getName()
+
+proc printLisp*(gState: State, root: TSNode): string =
+  var
+    node = root
+    nextnode: TSNode
+    depth = 0
+
+  while true:
+    if not node.isNil() and depth > -1:
+      if gState.pretty:
+        result &= spaces(depth)
+      let
+        (line, col) = gState.getLineCol(node)
+      result &= &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}"
+      let
+        val = gState.getNodeVal(node)
+      if "\n" notin val and " " notin val:
+        result &= &" \"{val}\""
+    else:
+      break
+
+    if node.tsNodeNamedChildCount() != 0:
+      if gState.pretty:
+        result &= "\n"
+      nextnode = node.tsNodeNamedChild(0)
+      depth += 1
+    else:
+      if gState.pretty:
+        result &= ")\n"
+      else:
+        result &= ")"
+      nextnode = node.tsNodeNextNamedSibling()
+
+    if nextnode.isNil():
+      while true:
+        node = node.tsNodeParent()
+        depth -= 1
+        if depth == -1:
+          break
+        if gState.pretty:
+          result &= spaces(depth) & ")\n"
+        else:
+          result &= ")"
+        if node == root:
+          break
+        if not node.tsNodeNextNamedSibling().isNil():
+          node = node.tsNodeNextNamedSibling()
+          break
+    else:
+      node = nextnode
+
+    if node == root:
+      break
+
+proc getCommented*(str: string): string =
+  "\n# " & str.strip().replace("\n", "\n# ")
+
+proc printTree*(nimState: NimState, pnode: PNode, offset = "") =
+  if nimState.gState.debug and pnode.kind != nkNone:
+    stdout.write "\n# " & offset & $pnode.kind & "("
+    case pnode.kind
+    of nkCharLit:
+      stdout.write "'" & pnode.intVal.char & "')"
+    of nkIntLit..nkUInt64Lit:
+      stdout.write $pnode.intVal & ")"
+    of nkFloatLit..nkFloat128Lit:
+      stdout.write $pnode.floatVal & ")"
+    of nkStrLit..nkTripleStrLit:
+      stdout.write "\"" & pnode.strVal & "\")"
+    of nkSym:
+      stdout.write $pnode.sym & ")"
+    of nkIdent:
+      stdout.write "\"" & $pnode.ident.s & "\")"
+    else:
+      if pnode.sons.len != 0:
+        for i in 0 ..< pnode.sons.len:
+          nimState.printTree(pnode.sons[i], offset & " ")
+          if i != pnode.sons.len - 1:
+            stdout.write ","
+        stdout.write "\n# " & offset & ")"
+      else:
+        stdout.write ")"
+    if offset.len == 0:
+      necho ""
+
+proc printDebug*(nimState: NimState, node: TSNode) =
+  if nimState.gState.debug:
+    necho ("Input => " & nimState.getNodeVal(node)).getCommented() & "\n" &
+          nimState.gState.printLisp(node).getCommented()
+
+proc printDebug*(nimState: NimState, pnode: PNode) =
+  if nimState.gState.debug:
+    necho ("Output => " & $pnode).getCommented()
+    nimState.printTree(pnode)
+
+# Compiler shortcuts
+
+proc getLineInfo*(nimState: NimState, node: TSNode): TLineInfo =
+  # Get Nim equivalent line:col info from node
+  let
+    (line, col) = nimState.gState.getLineCol(node)
+
+  result = newLineInfo(nimState.config, nimState.sourceFile.AbsoluteFile, line, col)
+
+proc getIdent*(nimState: NimState, name: string, info: TLineInfo, exported = true): PNode =
+  # Get ident PNode for name + info
+  let
+    exp = getIdent(nimState.identCache, "*")
+    ident = getIdent(nimState.identCache, name)
+
+  if exported:
+    result = newNode(nkPostfix)
+    result.add newIdentNode(exp, info)
+    result.add newIdentNode(ident, info)
+  else:
+    result = newIdentNode(ident, info)
+
+proc getNameInfo*(nimState: NimState, node: TSNode, kind: NimSymKind, parent = ""):
+  tuple[name: string, info: TLineInfo] =
+  # Shortcut to get identifier name and info (node value and line:col)
+  let
+    name = nimState.getNodeVal(node)
+  result.name = nimState.getIdentifier(name, kind, parent)
+  if kind == nskType:
+    result.name = result.name.getType()
+  result.info = nimState.getLineInfo(node)
+
 proc getCurrentHeader*(fullpath: string): string =
   ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")]))
 
@@ -291,91 +566,6 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string =
     replace(re"__attribute__[ ]*\(\(.*?\)\)([ ,;])", "$1").
     removeStatic()
 
-converter toString*(kind: Kind): string =
-  return case kind:
-    of exactlyOne:
-      ""
-    of oneOrMore:
-      "+"
-    of zeroOrMore:
-      "*"
-    of zeroOrOne:
-      "?"
-    of orWithNext:
-      "!"
-
-converter toKind*(kind: string): Kind =
-  return case kind:
-    of "+":
-      oneOrMore
-    of "*":
-      zeroOrMore
-    of "?":
-      zeroOrOne
-    of "!":
-      orWithNext
-    else:
-      exactlyOne
-
-proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool] =
-  if name[0] == '^':
-    result.recursive = true
-    result.name = name[1 .. ^1]
-  else:
-    result.name = name
-  result.kind = $name[^1]
-
-  if result.kind != exactlyOne:
-    result.name = result.name[0 .. ^2]
-
-proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
-  if node.tsNodeNamedChildCount() != 0:
-    for i in 0 .. node.tsNodeNamedChildCount()-1:
-      if $node.tsNodeType() != "comment":
-        result += 1
-
-proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
-  if node.tsNodeNamedChildCount() != 0:
-    for i in 0 .. node.tsNodeNamedChildCount()-1:
-      let
-        name = $node.tsNodeNamedChild(i).tsNodeType()
-
-      if name != "comment":
-        result.add(name)
-
-proc getRegexForAstChildren*(ast: ref Ast): string =
-  result = "^"
-  for i in 0 .. ast.children.len-1:
-    let
-      kind: string = ast.children[i].kind
-      begin = if result[^1] == '|': "" else: "(?:"
-    case kind:
-      of "!":
-        result &= &"{begin}{ast.children[i].name}|"
-      else:
-        result &= &"{begin}{ast.children[i].name}){kind}"
-  result &= "$"
-
-proc getAstChildByName*(ast: ref Ast, name: string): ref Ast =
-  for i in 0 .. ast.children.len-1:
-    if name in ast.children[i].name.split("|"):
-      return ast.children[i]
-
-  if ast.children.len == 1 and ast.children[0].name == ".":
-    return ast.children[0]
-
-proc getPxName*(node: TSNode, offset: int): string =
-  var
-    np = node
-    count = 0
-
-  while not np.tsNodeIsNull() and count < offset:
-    np = np.tsNodeParent()
-    count += 1
-
-  if count == offset and not np.tsNodeIsNull():
-    return $np.tsNodeType()
-
 proc getNimExpression*(nimState: NimState, expr: string): string =
   var
     clean = expr.multiReplace([("\n", " "), ("\r", "")])
@@ -435,42 +625,6 @@ proc getSplitComma*(joined: seq[string]): seq[string] =
   for i in joined:
     result = result.concat(i.split(","))
 
-proc getHeader*(nimState: NimState): string =
-  result =
-    if nimState.gState.dynlib.Bl:
-      &", header: {nimState.currentHeader}"
-    else:
-      ""
-
-proc getDynlib*(nimState: NimState): string =
-  result =
-    if nimState.gState.dynlib.nBl:
-      &", dynlib: {nimState.gState.dynlib}"
-    else:
-      ""
-
-proc getImportC*(nimState: NimState, origName, nimName: string): string =
-  if nimName != origName:
-    result = &"importc: \"{origName}\"{nimState.getHeader()}"
-  else:
-    result = nimState.impShort
-
-proc getPragma*(nimState: NimState, pragmas: varargs[string]): string =
-  result = ""
-  for pragma in pragmas.items():
-    if pragma.nBl:
-      result &= pragma & ", "
-  if result.nBl:
-    result = " {." & result[0 .. ^3] & ".}"
-
-  result = result.replace(nimState.impShort & ", cdecl", nimState.impShort & "C")
-
-  let
-    dy = nimState.getDynlib()
-
-  if ", cdecl" in result and dy.nBl:
-    result = result.replace(".}", dy & ".}")
-
 proc getComments*(nimState: NimState, strip = false): string =
   if not nimState.gState.nocomments and nimState.commentStr.nBl:
     result = "\n" & nimState.commentStr
@@ -505,6 +659,6 @@ proc loadPlugin*(gState: State, sourcePath: string) =
 
 proc expandSymlinkAbs*(path: string): string =
   try:
-    result = path.expandSymlink().absolutePath(path.parentDir()).myNormalizedPath()
+    result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath()
   except:
     result = path

nimterop/globals.nim

@@ -5,6 +5,7 @@ import regex
 import "."/plugin
 
 when not declared(CIMPORT):
+  import compiler/[ast, idents, options]
   import "."/treesitter/api
 
 const
@@ -34,24 +35,6 @@ const
   ].concat(toSeq(gExpressions.items))
 
 type
-  Kind = enum
-    exactlyOne
-    oneOrMore     # +
-    zeroOrMore    # *
-    zeroOrOne     # ?
-    orWithNext    # !
-
-  Ast = object
-    name*: string
-    kind*: Kind
-    recursive*: bool
-    children*: seq[ref Ast]
-    when not declared(CIMPORT):
-      tonim*: proc (ast: ref Ast, node: TSNode, nimState: NimState)
-    regex*: Regex
-
-  AstTable {.used.} = TableRef[string, seq[ref Ast]]
-
   State = ref object
     compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string]
 
@@ -67,7 +50,13 @@ type
   NimState {.used.} = ref object
     identifiers*: TableRef[string, string]
 
-    commentStr*, constStr*, debugStr*, enumStr*, procStr*, skipStr*, typeStr*: string
+    commentStr*, debugStr*, skipStr*: string
+
+    # Nim compiler objects
+    when not declared(CIMPORT):
+      constSection*, enumSection*, procSection*, typeSection*: PNode
+      identCache*: IdentCache
+      config*: ConfigRef
 
     gState*: State
 
@@ -94,7 +83,7 @@ type CompileMode = enum
 const modeDefault {.used.} = $cpp # TODO: USE this everywhere relevant
 
 when not declared(CIMPORT):
-  export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState,
+  export gAtoms, gExpressions, gEnumVals, State, NimState,
     nBl, Bl, CompileMode, modeDefault
 
   # Redirect output to file when required
@@ -107,3 +96,7 @@ when not declared(CIMPORT):
   template necho*(args: string) {.dirty.} =
     let gState = nimState.gState
     gecho args
+
+  template decho*(str: untyped): untyped =
+    if nimState.gState.debug:
+      necho str.getCommented()

nimterop/setup.nim

@@ -6,7 +6,7 @@ proc treesitterSetup*() =
   gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """
 lib/include/*
 lib/src/*
-""", "0.15.5")
+""", "0.16.2")
 
   gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """
 *.c
@@ -36,7 +36,7 @@ src/*.h
 src/*.c
 src/*.cc
 src/tree_sitter/parser.h
-""", "v0.15.0")
+""", "v0.16.0")
 
   writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """
 const TSLanguage *tree_sitter_c();
@@ -48,7 +48,7 @@ src/*.h
 src/*.c
 src/*.cc
 src/tree_sitter/parser.h
-""", "v0.15.0")
+""", "v0.16.0")
 
   writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """
 const TSLanguage *tree_sitter_cpp();

nimterop/toast.nim

@@ -2,58 +2,9 @@ import os, osproc, strformat, strutils, times
 
 import "."/treesitter/[api, c, cpp]
 
-import "."/[ast, compat, globals, getters, grammar]
+import "."/[ast2, globals, getters]
 
-proc printLisp(gState: State, root: TSNode) =
-  var
-    node = root
-    nextnode: TSNode
-    depth = 0
-
-  while true:
-    if not node.tsNodeIsNull() and depth > -1:
-      if gState.pretty:
-        stdout.write spaces(depth)
-      let
-        (line, col) = gState.getLineCol(node)
-      stdout.write &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}"
-    else:
-      break
-
-    if node.tsNodeNamedChildCount() != 0:
-      if gState.pretty:
-        gecho ""
-      nextnode = node.tsNodeNamedChild(0)
-      depth += 1
-    else:
-      if gState.pretty:
-        gecho ")"
-      else:
-        stdout.write ")"
-      nextnode = node.tsNodeNextNamedSibling()
-
-    if nextnode.tsNodeIsNull():
-      while true:
-        node = node.tsNodeParent()
-        depth -= 1
-        if depth == -1:
-          break
-        if gState.pretty:
-          gecho spaces(depth) & ")"
-        else:
-          stdout.write ")"
-        if node == root:
-          break
-        if not node.tsNodeNextNamedSibling().tsNodeIsNull():
-          node = node.tsNodeNextNamedSibling()
-          break
-    else:
-      node = nextnode
-
-    if node == root:
-      break
-
-proc process(gState: State, path: string, astTable: AstTable) =
+proc process(gState: State, path: string) =
   doAssert existsFile(path), &"Invalid path {path}"
 
   var
@@ -91,9 +42,9 @@ proc process(gState: State, path: string, astTable: AstTable) =
     tree.tsTreeDelete()
 
   if gState.past:
-    gState.printLisp(root)
+    gecho gState.printLisp(root)
   elif gState.pnim:
-    gState.printNim(path, root, astTable)
+    gState.printNim(path, root)
   elif gState.preprocess:
     gecho gState.code
 
@@ -109,7 +60,6 @@ proc main(
     nocomments = false,
     output = "",
     past = false,
-    pgrammar = false,
     pluginSourcePath: string = "",
     pnim = false,
     prefix: seq[string] = @[],
@@ -167,19 +117,12 @@ proc main(
     doAssert gState.outputHandle.open(outputFile, fmWrite),
       &"Failed to write to {outputFile}"
 
-  # Process grammar into AST
-  let
-    astTable = parseGrammar()
-
-  if pgrammar:
-    # Print AST of grammar
-    gState.printGrammar(astTable)
-  elif source.nBl:
+  if source.nBl:
     # Print source after preprocess or Nim output
     if gState.pnim:
       gState.printNimHeader()
     for src in source:
-      gState.process(src.expandSymlinkAbs(), astTable)
+      gState.process(src.expandSymlinkAbs())
 
   # Close outputFile
   if outputFile.len != 0:
@@ -219,9 +162,9 @@ proc main(
 
         # Rerun nim check on stubbed wrapper
         (check, err) = execCmdEx(&"{gState.nim} check {outputFile}")
-        doAssert err == 0, data & "# Nim check with stub failed:\n\n" & check
+        doAssert err == 0, data & "\n# Nim check with stub failed:\n\n" & check
       else:
-        doAssert err == 0, outputFile.readFile() & "# Nim check failed:\n\n" & check
+        doAssert err == 0, outputFile.readFile() & "\n# Nim check failed:\n\n" & check
 
   # Print wrapper if temporarily redirected to file
   if check and output.len == 0:
@@ -241,7 +184,6 @@ when isMainModule:
     "nocomments": "exclude top-level comments from output",
     "output": "file to output content - default stdout",
     "past": "print AST output",
-    "pgrammar": "print grammar",
     "pluginSourcePath": "Nim file to build and load as a plugin",
     "pnim": "print Nim output",
     "preprocess": "run preprocessor on header",
@@ -260,7 +202,6 @@ when isMainModule:
     "nocomments": 'c',
     "output": 'o',
     "past": 'a',
-    "pgrammar": 'g',
     "pnim": 'n',
     "prefix": 'E',
     "preprocess": 'p',

nimterop/types.nim

@@ -1,26 +1,7 @@
-# see https://github.com/nimterop/nimterop/issues/79
-
-when (NimMajor, NimMinor, NimPatch) < (0, 19, 9):
-  # clean this up once upgraded; adapted from std/time_t
-  when defined(nimdoc):
-    type
-      impl = distinct int64
-      Time = impl
-  elif defined(windows):
-    when defined(i386) and defined(gcc):
-      type Time {.importc: "time_t", header: "<time.h>".} = distinct int32
-    else:
-      type Time {.importc: "time_t", header: "<time.h>".} = distinct int64
-  elif defined(posix):
-    import posix
-  type
-    time_t* = Time
-    time64_t* = Time
-else:
-  import std/time_t as time_t_temp
-  type
-    time_t* = time_t_temp.Time
-    time64_t* = time_t_temp.Time
+import std/time_t as time_t_temp
+type
+  time_t* = time_t_temp.Time
+  time64_t* = time_t_temp.Time
 
 when defined(cpp):
   # http://www.cplusplus.com/reference/cwchar/wchar_t/

tests/include/test3.h

@@ -0,0 +1,42 @@
+
+#define A 1
+#define B 1.0
+#define C 0x10
+#define D "hello"
+#define E 'c'
+
+struct A0;
+struct A1 {};
+typedef struct A2;
+typedef struct A3 {};
+typedef struct A4 A4, *A4p;
+typedef const int A5;
+typedef int *A6;
+typedef A0 **A7;
+typedef void *A8;
+
+typedef char *A9[3];
+typedef char *A10[3][6];
+typedef char *(*A11)[3];
+
+typedef int **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int));
+typedef int A13(int, int);
+
+struct A14 { volatile char a1; };
+struct A15 { char *a1; const int *a2[1]; };
+
+typedef struct A16 { char f1; };
+typedef struct A17 { char *a1; int *a2[1]; } A18, *A18p;
+typedef struct { char *a1; int *a2[1]; } A19, *A19p;
+
+typedef struct A20 { char a1; } A20, A21, *A21p;
+
+//Expression
+//typedef struct A21 { int **f1; int abc[123+132]; } A21;
+
+//Unions
+//union UNION1 {int f1; };
+//typedef union UNION2 { int **f1; int abc[123+132]; } UNION2;
+
+// Anonymous
+//typedef struct { char a1; };

tests/tnimterop2.nim

@@ -0,0 +1,71 @@
+import tables
+
+import nimterop/[cimport]
+
+static:
+  cDebug()
+
+cImport("include/test3.h", flags="-d")
+
+proc testFields(t: typedesc, fields: Table[string, string] = initTable[string, string]()) =
+  var
+    obj: t
+    count = 0
+  for name, value in obj.fieldPairs():
+    count += 1
+    assert name in fields, $t & "." & name & " invalid"
+    assert $fields[name] == $typeof(value),
+      "typeof(" & $t & ":" & name & ") != " & fields[name] & ", is " & $typeof(value)
+  assert count == fields.len, "Failed for " & $t
+
+assert A == 1
+assert B == 1.0
+assert C == 0x10
+assert D == "hello"
+assert E == 'c'
+
+assert A0 is object
+testFields(A0)
+assert A1 is object
+testFields(A1)
+assert A2 is object
+testFields(A2)
+assert A3 is object
+testFields(A3)
+assert A4 is object
+testFields(A4)
+assert A4p is ptr A4
+assert A5 is cint
+assert A6 is ptr cint
+assert A7 is ptr ptr A0
+assert A8 is pointer
+
+assert A9 is array[3, cstring]
+assert A10 is array[3, array[6, cstring]]
+assert A11 is ptr array[3, cstring]
+
+assert A12 is proc(a1: cint, b: cint, c: ptr cint, a4: ptr cint, count: array[4, ptr cint], `func`: proc(a1: cint, a2: cint): cint): ptr ptr cint
+assert A13 is proc(a1: cint, a2: cint): cint
+
+assert A14 is object
+testFields(A14, {"a1": "cchar"}.toTable())
+
+assert A15 is object
+testFields(A15, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable())
+
+assert A16 is object
+testFields(A16, {"f1": "cchar"}.toTable())
+
+assert A17 is object
+testFields(A17, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable())
+assert A18 is A17
+assert A18p is ptr A17
+
+assert A19 is object
+testFields(A19, {"a1": "cstring", "a2": "array[0..0, ptr cint]"}.toTable())
+assert A19p is ptr A19
+
+assert A20 is object
+testFields(A20, {"a1": "cchar"}.toTable())
+assert A21 is A20
+assert A21p is ptr A20