Multiple enhancements (#128)

Multiple enhancements

README.md

@@ -10,13 +10,14 @@ Nim has one of the best FFI you can find - importing C/C++ is supported out of t
 
 The goal of nimterop is to leverage the [tree-sitter](http://tree-sitter.github.io/tree-sitter/) engine to parse C/C++ code and then convert relevant portions of the AST into Nim definitions. [tree-sitter](https://github.com/tree-sitter) is a Github sponsored project that can parse a variety of languages into an AST which is then leveraged by the [Atom](https://atom.io/) editor for syntax highlighting and code folding. The advantages of this approach are multifold:
 - Benefit from the tree-sitter community's investment into language parsing
+- Wrap what is recognized in the AST rather than completely failing due to parsing errors
 - Avoid depending on Nim compiler API which is evolving constantly and makes backwards compatibility a bit challenging
 
 Most of the functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how c2nim can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application.
 
 The nimterop feature set is still limited to C but is expanding rapidly. C++ support will be added once most popular C libraries can be wrapped seamlessly.
 
-Given the simplicity and success of this approach so far, it seems feasible to continue on for more complex code. The goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation.
+Nimterop has seen some adoption within the community and the simplicity and success of this approach justifies additional investment of time and effort. Regardless, the goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation.
 
 __Installation__
 
@@ -57,15 +58,15 @@ __Implementation Details__
 
 In order to use the tree-sitter C library, it has to be compiled into a separate binary called `toast` (to AST) since the Nim VM doesn't yet support FFI. `toast` takes a C/C++ file and runs it through the tree-sitter API which returns an AST data structure. This can then be printed out to stdout in a Lisp S-Expression format or the relevant Nim wrapper output. This content can be saved to a `.nim` file and imported if so desired.
 
-Alternatively, the `cImport()` macro allows easier creation of wrappers in code. It runs `toast` on the specified header file and injects the generated wrapper content into the application at compile time. A few other helper procs are provided to influence this process.
+Alternatively, the `cImport()` macro allows easier creation of wrappers in code. It runs `toast` on the specified header file and injects the generated wrapper content into the application at compile time. A few other helper procs are provided to influence this process. Output is cached to save time on subsequent runs.
 
-`toast` can also be used to run the header through the preprocessor which cleans up the code considerably. Along with the recursion capability which runs through all #include files, one large simpler header file can be created which can then be processed with c2nim if so desired.
+`toast` can also be used to run the header through the preprocessor which cleans up the code considerably. Along with the recursion capability which runs through all #include files, one large simpler header file can be created which can then be processed with `toast` or even `c2nim` if so desired.
 
 The tree-sitter library is limited as well - it may fail on some advanced language constructs but is designed to handle them gracefully since it is expected to have bad code while actively typing in an editor. When an error is detected, tree-sitter includes an ERROR node at that location in the AST. At this time, `cImport()` will complain and continue if it encounters any errors. Depending on how severe the errors are, compilation may succeed or fail. Glaring issues will be communicated to the tree-sitter team but their goals may not always align with those of this project.
 
 __Credits__
 
-Nimterop depends on [tree-sitter](http://tree-sitter.github.io/tree-sitter/) and all licensing terms of [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/master/LICENSE) apply to the usage of this package. Interestingly, the tree-sitter functionality is [wrapped](https://github.com/genotrance/nimtreesitter) using c2nim and nimgen at this time. Depending on the success of this project, those could perhaps be bootstrapped using nimterop eventually.
+Nimterop depends on [tree-sitter](http://tree-sitter.github.io/tree-sitter/) and all licensing terms of [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/master/LICENSE) apply to the usage of this package. The tree-sitter functionality is pulled and wrapped using nimterop itself.
 
 __Feedback__
 

nimterop.nimble

@@ -27,12 +27,9 @@ proc execTest(test: string) =
 proc tsoloud() =
   execTest "tests/tsoloud.nim"
 
-proc buildToast() =
+task buildToast, "build toast":
-  execCmd(&"nim c -d:release nimterop/toast.nim")
-
-task rebuildToast, "rebuild toast":
   # If need to manually rebuild (automatically built on 1st need)
-  buildToast()
+  execCmd(&"nim c -d:release nimterop/toast.nim")
 
 proc testAll() =
   execTest "tests/tnimterop_c.nim"
@@ -69,7 +66,7 @@ proc runNimDoc() =
     execCmd &"nim js -o:{htmldocsDir}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim"
 
 task test, "Test":
-  buildToast()
+  buildToastTask()
   testAll()
   runNimDoc()
 

nimterop/ast.nim

@@ -8,7 +8,7 @@ proc saveNodeData(node: TSNode, nimState: NimState): bool =
   let name = $node.tsNodeType()
   if name in gAtoms:
     var
-      val = node.getNodeVal()
+      val = nimState.getNodeVal(node)
 
     if name == "primitive_type" and node.tsNodeParent.tsNodeType() == "sized_type_specifier":
       return true
@@ -52,9 +52,9 @@ proc saveNodeData(node: TSNode, nimState: NimState): bool =
           nimState.data.insert(("pointer_declarator", ""), nimState.data.len-1)
       nimState.data.add(("function_declarator", ""))
 
-  elif name in gExpressions:
+  elif name in gExpressions and name != "escape_sequence":
     if $node.tsNodeParent.tsNodeType() notin gExpressions:
-      nimState.data.add((name, node.getNodeVal()))
+      nimState.data.add((name, nimState.getNodeVal(node)))
 
   elif name in ["abstract_pointer_declarator", "enumerator", "field_declaration", "function_declarator"]:
     nimState.data.add((name.replace("abstract_", ""), ""))
@@ -107,7 +107,7 @@ proc searchAst(root: TSNode, astTable: AstTable, nimState: NimState) =
         for ast in astTable[name]:
           if searchAstForNode(ast, node, nimState):
             ast.tonim(ast, node, nimState)
-            if gStateRT.debug:
+            if nimState.gState.debug:
               nimState.debugStr &= "\n# " & nimState.data.join("\n# ") & "\n"
             break
         nimState.data = @[]
@@ -142,31 +142,37 @@ proc printNimHeader*() =
 # Command line:
 #   $2 $3
 
-{.experimental: "codeReordering".}
 {.hint[ConvFromXtoItselfNotNeeded]: off.}
 
 import nimterop/types
 """ % [$now(), getAppFilename(), commandLineParams().join(" ")]
 
-proc printNim*(fullpath: string, root: TSNode, astTable: AstTable) =
+proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) =
   var
     nimState = new(NimState)
     fp = fullpath.replace("\\", "/")
   nimState.identifiers = newTable[string, string]()
 
   nimState.currentHeader = getCurrentHeader(fullpath)
+  nimState.impHeader = nimState.currentHeader.replace("header", "imp")
+  nimState.sourceFile = fullpath
   nimState.constStr &= &"\n  {nimState.currentHeader} {{.used.}} = \"{fp}\""
 
-  nimState.debug = gStateRT.debug
+  nimState.gState = gState
 
   root.searchAst(astTable, nimState)
 
   if nimState.enumStr.nBl:
-    echo nimState.enumStr
+    echo &"{nimState.enumStr}\n"
 
   if nimState.constStr.nBl:
     echo &"const{nimState.constStr}\n"
 
+  echo &"""
+{{.pragma: {nimState.impHeader}, importc, header: {nimState.currentHeader}.}}
+{{.pragma: {nimState.impHeader}C, {nimState.impHeader}, cdecl.}}
+"""
+
   if nimState.typeStr.nBl:
     echo &"type{nimState.typeStr}\n"
 

nimterop/astold.nim

@@ -1,297 +0,0 @@
-import macros, os, strformat, strutils
-
-import treesitter/api
-
-import getters, globals
-
-#
-# Preprocessor
-#
-
-proc pPreprocDef(node: TSNode) =
-  if node.tsNodeNamedChildCount() == 2:
-    let
-      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
-      val = getNodeValIf(node.tsNodeNamedChild(1), "preproc_arg")
-
-    if name.nBl and val.nBl and name notin gStateRT.consts:
-      gStateRT.consts.add(name)
-      if val.getLit().nBl:
-        # #define NAME VALUE
-        gStateRT.constStr &= &"  {name.getIdentifier()}* = {val}        # pPreprocDef()\n"
-
-#
-# Types
-#
-
-proc typeScan(node: TSNode, sym, id: string, offset: string): string =
-  if node.tsNodeIsNull() or $node.tsNodeType() != sym or node.tsNodeNamedChildCount() != 2:
-    return
-
-  var
-    name = getNodeValIf(node.tsNodeNamedChild(1), id)
-    ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-    ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    ptrname = false
-
-  if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
-    name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), id)
-    ptrname = true
-
-  if name.len == 0:
-    return
-  elif ptyp.nBl:
-    ptyp = ptyp.getType()
-    if ptyp != "object" and ptrname:
-      ptyp = &"ptr {ptyp}"
-    result = &"{offset}{name.getIdentifier()}: {ptyp}"
-  elif ttyp.nBl:
-    if ptrname:
-      ttyp = &"ptr {ttyp}"
-    result = &"{offset}{name.getIdentifier()}: {ttyp}"
-  elif $node.tsNodeNamedChild(0).tsNodeType() in ["struct_specifier", "enum_specifier"] and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
-    var styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
-    if styp.nBl:
-      if ptrname:
-        styp = &"ptr {styp}"
-      result = &"{offset}{name.getIdentifier()}: {styp}"
-
-proc pStructSpecifier(node: TSNode, name = "") =
-  var stmt: string
-  if node.tsNodeNamedChildCount() == 1 and name notin gStateRT.types:
-    case $node.tsNodeNamedChild(0).tsNodeType():
-      of "type_identifier":
-        let typ = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-        if typ.nBl:
-          gStateRT.types.add(name)
-          if name != typ:
-            # typedef struct X Y
-            gStateRT.typeStr &= &"  {name.getIdentifier()}* = {typ}        #1 pStructSpecifier()\n"
-          else:
-            # typedef struct X X
-            gStateRT.typeStr &= &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #2 pStructSpecifier()\n"
-
-      of "field_declaration_list":
-        # typedef struct { fields } X
-        stmt = &"  {name.getIdentifier()}* {{.importc: \"{name}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #3 pStructSpecifier()\n"
-
-        if node.tsNodeNamedChild(0).tsNodeNamedChildCount() != 0:
-          for i in 0 .. node.tsNodeNamedChild(0).tsNodeNamedChildCount()-1:
-            if $node.tsNodeNamedChild(0).tsNodeNamedChild(i).tsNodeType() == "comment":
-              continue
-            let ts = typeScan(node.tsNodeNamedChild(0).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
-            if ts.len == 0:
-              return
-            stmt &= ts & "\n"
-
-          gStateRT.types.add(name)
-          gStateRT.typeStr &= stmt
-      else:
-        discard
-
-  elif name.len == 0 and node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "field_declaration_list":
-    let ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    if ename.nBl and ename notin gStateRT.types:
-      # struct X { fields }
-      stmt &= &"  {ename}* {{.importc: \"struct {ename}\", header: {gStateRT.currentHeader}, bycopy.}} = object        #4 pStructSpecifier()\n"
-
-      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
-        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
-          if $node.tsNodeNamedChild(1).tsNodeNamedChild(i).tsNodeType() == "comment":
-            continue
-          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "field_declaration", "field_identifier", "    ")
-          if ts.len == 0:
-            return
-          stmt &= ts & "\n"
-
-        gStateRT.types.add(name)
-        gStateRT.typeStr &= stmt
-
-proc pEnumSpecifier(node: TSNode, name = "") =
-  var
-    ename: string
-    elid: uint32
-    stmt: string
-
-  if node.tsNodeNamedChildCount() == 1 and $node.tsNodeNamedChild(0).tsNodeType() == "enumerator_list":
-    # typedef enum { fields } X
-    ename = name
-    elid = 0
-    stmt = &"  {name.getIdentifier()}* = enum        #1 pEnumSpecifier()\n"
-  elif node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "enumerator_list":
-    if name.len == 0:
-      ename = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-    else:
-      ename = name
-    elid = 1
-    if ename.nBl:
-      # enum X { fields }
-      stmt = &"  {ename}* = enum        #2 pEnumSpecifier()\n"
-    else:
-      return
-  else:
-    return
-
-  if node.tsNodeNamedChild(elid).tsNodeNamedChildCount() != 0:
-    for i in 0 .. node.tsNodeNamedChild(elid).tsNodeNamedChildCount()-1:
-      let field = node.tsNodeNamedChild(elid).tsNodeNamedChild(i)
-      if $field.tsNodeType() == "comment":
-        continue
-      if not field.tsNodeIsNull() and $field.tsNodeType() == "enumerator":
-        let fname = getNodeValIf(field.tsNodeNamedChild(0), "identifier")
-        if field.tsNodeNamedChildCount() == 1:
-          stmt &= &"    {fname}\n"
-        elif field.tsNodeNamedChildCount() == 2 and $field.tsNodeNamedChild(1).tsNodeType() == "number_literal":
-          let num = getNodeValIf(field.tsNodeNamedChild(1), "number_literal")
-          stmt &= &"    {fname} = {num}\n"
-        else:
-          return
-
-    if ename notin gStateRT.types:
-      gStateRT.types.add(name)
-      gStateRT.typeStr &= stmt
-
-proc pTypeDefinition(node: TSNode) =
-  if node.tsNodeNamedChildCount() == 2:
-    var
-      name = getNodeValIf(node.tsNodeNamedChild(1), "type_identifier")
-      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-      ptrname = false
-
-    if name.len == 0 and $node.tsNodeNamedChild(1).tsNodeType() == "pointer_declarator" and node.tsNodeNamedChild(1).tsNodeNamedChildCount() == 1:
-      name = getNodeValIf(node.tsNodeNamedChild(1).tsNodeNamedChild(0), "type_identifier")
-      ptrname = true
-
-    if name.nBl and name notin gStateRT.types:
-      if ptyp.nBl:
-        # typedef int X
-        gStateRT.types.add(name)
-        ptyp = ptyp.getType()
-        if ptyp != "object" and ptrname:
-          ptyp = &"ptr {ptyp}"
-        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ptyp}        #1 pTypeDefinition()\n"
-      elif ttyp.nBl:
-        # typedef X Y
-        gStateRT.types.add(name)
-        if ptrname:
-          ttyp = &"ptr {ttyp}"
-        gStateRT.typeStr &= &"  {name.getIdentifier()}* = {ttyp}        #2 pTypeDefinition()\n"
-      else:
-        case $node.tsNodeNamedChild(0).tsNodeType():
-          of "struct_specifier":
-            pStructSpecifier(node.tsNodeNamedChild(0), name)
-          of "enum_specifier":
-            pEnumSpecifier(node.tsNodeNamedChild(0), name)
-          else:
-            discard
-
-proc pFunctionDeclarator(node: TSNode, typ: string) =
-  if node.tsNodeNamedChildCount() == 2:
-    let
-      name = getNodeValIf(node.tsNodeNamedChild(0), "identifier")
-
-    if name.nBl and name notin gStateRT.procs and $node.tsNodeNamedChild(1).tsNodeType() == "parameter_list":
-      # typ function(typ param1, ...)
-      var stmt = &"# pFunctionDeclarator()\nproc {name.getIdentifier()}*("
-
-      if node.tsNodeNamedChild(1).tsNodeNamedChildCount() != 0:
-        for i in 0 .. node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
-          let ts = typeScan(node.tsNodeNamedChild(1).tsNodeNamedChild(i), "parameter_declaration", "identifier", "")
-          if ts.len == 0:
-            return
-          stmt &= ts
-          if i != node.tsNodeNamedChild(1).tsNodeNamedChildCount()-1:
-            stmt &= ", "
-
-      if typ != "void":
-        stmt &= &"): {typ.getType()} "
-      else:
-        stmt &= ") "
-
-      stmt &= &"{{.importc: \"{name}\", header: {gStateRT.currentHeader}.}}\n"
-
-      gStateRT.procs.add(name)
-      gStateRT.procStr &= stmt
-
-proc pDeclaration*(node: TSNode) =
-  if node.tsNodeNamedChildCount() == 2 and $node.tsNodeNamedChild(1).tsNodeType() == "function_declarator":
-    let
-      ptyp = getNodeValIf(node.tsNodeNamedChild(0), "primitive_type")
-      ttyp = getNodeValIf(node.tsNodeNamedChild(0), "type_identifier")
-
-    if ptyp.nBl:
-      pFunctionDeclarator(node.tsNodeNamedChild(1), ptyp.getType())
-    elif ttyp.nBl:
-      pFunctionDeclarator(node.tsNodeNamedChild(1), ttyp)
-    elif $node.tsNodeNamedChild(0).tsNodeType() == "struct_specifier" and node.tsNodeNamedChild(0).tsNodeNamedChildCount() == 1:
-      let styp = getNodeValIf(node.tsNodeNamedChild(0).tsNodeNamedChild(0), "type_identifier")
-      if styp.nBl:
-        pFunctionDeclarator(node.tsNodeNamedChild(1), styp)
-
-proc genNimAst(root: TSNode) =
-  var
-    node = root
-    nextnode: TSNode
-
-  while true:
-    if not node.tsNodeIsNull():
-      case $node.tsNodeType():
-        of "ERROR":
-          let (line, col) = getLineCol(node)
-          let file = gStateRT.sourceFile
-          echo &"# [toast] Potentially invalid syntax at {file}:{line}:{col}"
-        of "preproc_def":
-          pPreprocDef(node)
-        of "type_definition":
-          pTypeDefinition(node)
-        of "declaration":
-          pDeclaration(node)
-        of "struct_specifier":
-          if $node.tsNodeParent().tsNodeType() notin ["type_definition", "declaration"]:
-            pStructSpecifier(node)
-        of "enum_specifier":
-          if $node.tsNodeParent.tsNodeType() notin ["type_definition", "declaration"]:
-            pEnumSpecifier(node)
-        else:
-          # TODO: log
-          discard
-    else:
-      return
-
-    if node.tsNodeNamedChildCount() != 0:
-      nextnode = node.tsNodeNamedChild(0)
-    else:
-      nextnode = node.tsNodeNextNamedSibling()
-
-    if nextnode.tsNodeIsNull():
-      while true:
-        node = node.tsNodeParent()
-        if node == root:
-          break
-        if not node.tsNodeNextNamedSibling().tsNodeIsNull():
-          node = node.tsNodeNextNamedSibling()
-          break
-    else:
-      node = nextnode
-
-    if node == root:
-      break
-
-proc printNim*(fullpath: string, root: TSNode) =
-  echo "{.experimental: \"codeReordering\".}"
-
-  var fp = fullpath.replace("\\", "/")
-  gStateRT.currentHeader = getCurrentHeader(fullpath)
-  gStateRT.constStr &= &"  {gStateRT.currentHeader} = \"{fp}\"\n"
-
-  genNimAst(root)
-
-  if gStateRT.constStr.nBl:
-    echo "const\n" & gStateRT.constStr
-
-  if gStateRT.typeStr.nBl:
-    echo "type\n" & gStateRT.typeStr
-
-  if gStateRT.procStr.nBl:
-    echo gStateRT.procStr

nimterop/cimport.nim

@@ -138,7 +138,7 @@ proc getToast(fullpath: string, recurse: bool = false): string =
     cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.quoteShell}"
 
   cmd.add &" {fullpath.quoteShell}"
-  echo "# " & cmd
+
   # see https://github.com/nimterop/nimterop/issues/69
   (result, ret) = gorgeEx(cmd, cache=getCacheValue(fullpath))
   doAssert ret == 0, getToastError(result)
@@ -195,7 +195,7 @@ macro cOverride*(body): untyped =
   result = body
 
   if gStateCT.debug:
-    echo "Overriding " & gStateCT.symOverride.join(" ")
+    echo "# Overriding " & gStateCT.symOverride.join(" ")
 
 proc cSkipSymbol*(skips: seq[string]) {.compileTime.} =
   ## Similar to `cOverride() <cimport.html#cOverride.m,>`_, this macro allows
@@ -320,7 +320,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped =
       {.passC: `str`.}
 
     if gStateCT.debug:
-      echo result.repr
+      echo result.repr & "\n"
 
 proc cAddSearchDir*(dir: string) {.compileTime.} =
   ## Add directory ``dir`` to the search path used in calls to
@@ -483,13 +483,13 @@ macro cImport*(filename: static string, recurse: static bool = false): untyped =
   let
     output = getToast(fullpath, recurse)
 
+  if gStateCT.debug:
+    echo output
+
   try:
     let body = parseStmt(output)
 
     result.add body
-
-    if gStateCT.debug:
-      echo result.repr
   except:
     let
       (tmpFile, errors) = getNimCheckError(output)

nimterop/getters.nim

@@ -104,14 +104,14 @@ proc checkIdentifier(name, kind, parent, origName: string) =
   if parent.nBl:
     doAssert name.nBl, &"Blank identifier, originally '{parentStr}{name}' ({kind}), cannot be empty"
 
-proc getIdentifier*(name: string, kind: NimSymKind, parent=""): string =
+proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent=""): string =
   doAssert name.len != 0, "Blank identifier error"
 
-  if name notin gStateRT.symOverride or parent.nBl:
+  if name notin nimState.gState.symOverride or parent.nBl:
-    if gStateRT.onSymbol != nil:
+    if nimState.gState.onSymbol != nil:
       var
         sym = Symbol(name: name, parent: parent, kind: kind)
-      gStateRT.onSymbol(sym)
+      nimState.gState.onSymbol(sym)
 
       result = sym.name
     else:
@@ -124,27 +124,27 @@ proc getIdentifier*(name: string, kind: NimSymKind, parent=""): string =
   else:
     result = ""
 
-proc getUniqueIdentifier*(existing: TableRef[string, string], prefix = ""): string =
+proc getUniqueIdentifier*(nimState: NimState, prefix = ""): string =
   var
-    name = prefix & "_" & gStateRT.sourceFile.extractFilename().multiReplace([(".", ""), ("-", "")])
+    name = prefix & "_" & nimState.sourceFile.extractFilename().multiReplace([(".", ""), ("-", "")])
     nimName = name[0] & name[1 .. ^1].replace("_", "").toLowerAscii
     count = 1
 
-  while (nimName & $count) in existing:
+  while (nimName & $count) in nimState.identifiers:
     count += 1
 
   return name & $count
 
-proc addNewIdentifer*(existing: var TableRef[string, string], name: string): bool =
+proc addNewIdentifer*(nimState: NimState, name: string): bool =
-  if name notin gStateRT.symOverride:
+  if name notin nimState.gState.symOverride:
     let
       nimName = name[0] & name[1 .. ^1].replace("_", "").toLowerAscii
 
-    if existing.hasKey(nimName):
+    if nimState.identifiers.hasKey(nimName):
-      doAssert name == existing[nimName], &"Identifier '{name}' is a stylistic duplicate of identifier '{existing[nimName]}', use 'cPlugin:onSymbol()' to rename"
+      doAssert name == nimState.identifiers[nimName], &"Identifier '{name}' is a stylistic duplicate of identifier '{nimState.identifiers[nimName]}', use 'cPlugin:onSymbol()' to rename"
       result = false
     else:
-      existing[nimName] = name
+      nimState.identifiers[nimName] = name
       result = true
 
 proc getPtrType*(str: string): string =
@@ -166,20 +166,14 @@ proc getLit*(str: string): string =
     str.contains(re"^0x[\d]+$"):
     return str
 
-proc getNodeVal*(node: TSNode): string =
+proc getNodeVal*(nimState: NimState, node: TSNode): string =
-  return gStateRT.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
+  return nimState.gState.code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip()
 
-proc getNodeValIf*(node: TSNode, esym: string): string =
+proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
-  if esym != $node.tsNodeType():
-    return
-
-  return node.getNodeVal()
-
-proc getLineCol*(node: TSNode): tuple[line, col: int] =
   result.line = 1
   result.col = 1
   for i in 0 .. node.tsNodeStartByte().int-1:
-    if gStateRT.code[i] == '\n':
+    if gState.code[i] == '\n':
       result.col = 0
       result.line += 1
     result.col += 1
@@ -201,19 +195,19 @@ proc removeStatic(content: string): string =
       result.add(body.replace(re"(?m)^(.*\n?)", "//$1"))
   )
 
-proc getPreprocessor*(fullpath: string, mode = "cpp"): string =
+proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string =
   var
     mmode = if mode == "cpp": "c++" else: mode
-    cmd = &"gcc -E -dD -x{mmode} -w "
+    cmd = &"gcc -E -CC -dD -x{mmode} -w "
 
     rdata: seq[string] = @[]
     start = false
     sfile = fullpath.sanitizePath
 
-  for inc in gStateRT.includeDirs:
+  for inc in gState.includeDirs:
     cmd &= &"-I{inc.quoteShell} "
 
-  for def in gStateRT.defines:
+  for def in gState.defines:
     cmd &= &"-D{def} "
 
   cmd &= &"{fullpath.quoteShell}"
@@ -229,13 +223,13 @@ proc getPreprocessor*(fullpath: string, mode = "cpp"): string =
           start = true
         elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line:
           start = true
-        elif gStateRT.recurse:
+        elif gState.recurse:
           let
             pDir = sfile.expandFilename().parentDir().sanitizePath()
           if pDir.len == 0 or pDir in saniLine:
             start = true
           else:
-            for inc in gStateRT.includeDirs:
+            for inc in gState.includeDirs:
               if inc.absolutePath().sanitizePath in saniLine:
                 start = true
                 break
@@ -350,7 +344,7 @@ proc dll*(path: string): string =
 
   result = dir / (DynlibFormat % name)
 
-proc loadPlugin*(sourcePath: string) =
+proc loadPlugin*(gState: State, sourcePath: string) =
   doAssert fileExists(sourcePath), "Plugin file does not exist: " & sourcePath
 
   let
@@ -363,5 +357,5 @@ proc loadPlugin*(sourcePath: string) =
   let lib = loadLib(pdll)
   doAssert lib != nil, "Plugin $1 compiled to $2 failed to load" % [sourcePath, pdll]
 
-  gStateRT.onSymbol = cast[OnSymbol](lib.symAddr("onSymbol"))
+  gState.onSymbol = cast[OnSymbol](lib.symAddr("onSymbol"))
-  doAssert gStateRT.onSymbol != nil, "onSymbol() load failed from " & pdll
+  doAssert gState.onSymbol != nil, "onSymbol() load failed from " & pdll

nimterop/git.nim

@@ -54,7 +54,7 @@ proc extractZip*(zipfile, outdir: string) =
           "'System.IO.Compression.FileSystem'; " &
           "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\""
 
-  echo "Extracting " & zipfile
+  echo "# Extracting " & zipfile
   discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}")
 
 proc downloadUrl*(url, outdir: string) =
@@ -63,7 +63,7 @@ proc downloadUrl*(url, outdir: string) =
     ext = file.splitFile().ext.toLowerAscii()
 
   if not (ext == ".zip" and fileExists(outdir/file)):
-    echo "Downloading " & file
+    echo "# Downloading " & file
     mkDir(outdir)
     var cmd = if defined(Windows):
       "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#"
@@ -75,20 +75,20 @@ proc downloadUrl*(url, outdir: string) =
       extractZip(file, outdir)
 
 proc gitReset*(outdir: string) =
-  echo "Resetting " & outdir
+  echo "# Resetting " & outdir
 
   let cmd = &"cd {outdir.quoteShell} && git reset --hard"
   while execAction(cmd).contains("Permission denied"):
     sleep(1000)
-    echo "  Retrying ..."
+    echo "#   Retrying ..."
 
 proc gitCheckout*(file, outdir: string) =
-  echo "Resetting " & file
+  echo "# Resetting " & file
   let file2 = file.relativePath outdir
   let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}"
   while execAction(cmd).contains("Permission denied"):
     sleep(500)
-    echo "  Retrying ..."
+    echo "#   Retrying ..."
 
 proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
   if dirExists(outdir/".git"):
@@ -100,7 +100,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
 
   mkDir(outdir)
 
-  echo "Setting up Git repo: " & url
+  echo "# Setting up Git repo: " & url
   discard execAction(&"cd {outdirQ} && git init .")
   discard execAction(&"cd {outdirQ} && git remote add origin {url}")
 
@@ -112,27 +112,27 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
     writeFile(sparsefile, plist)
 
   if checkout.len != 0:
-    echo "Checking out " & checkout
+    echo "# Checking out " & checkout
     discard execAction(&"cd {outdirQ} && git pull --tags origin master")
     discard execAction(&"cd {outdirQ} && git checkout {checkout}")
   else:
-    echo "Pulling repository"
+    echo "# Pulling repository"
     discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master")
 
 proc configure*(path, check: string) =
   if (path / check).fileExists():
     return
 
-  echo "Configuring " & path
+  echo "# Configuring " & path
 
   if not fileExists(path / "configure"):
     if fileExists(path / "autogen.sh"):
-      echo "  Running autogen.sh"
+      echo "#   Running autogen.sh"
 
       discard execAction(&"cd {path.quoteShell} && bash autogen.sh")
 
   if fileExists(path / "configure"):
-    echo "  Running configure"
+    echo "#   Running configure"
 
     discard execAction(&"cd {path.quoteShell} && bash configure")
 

nimterop/globals.nim

@@ -12,6 +12,7 @@ const
     "field_identifier",
     "identifier",
     "number_literal",
+    "char_literal",
     "preproc_arg",
     "primitive_type",
     "sized_type_specifier",
@@ -22,12 +23,14 @@ const
     "parenthesized_expression",
     "bitwise_expression",
     "shift_expression",
-    "math_expression"
+    "math_expression",
+    "escape_sequence"
   ].toSet()
 
   gEnumVals {.used.} = @[
     "identifier",
-    "number_literal"
+    "number_literal",
+    "char_literal"
   ].concat(toSeq(gExpressions.items))
 
 type
@@ -49,29 +52,28 @@ type
 
   AstTable {.used.} = TableRef[string, seq[ref Ast]]
 
-  State = object
+  State = ref object
     compile*, defines*, headers*, includeDirs*, searchDirs*, symOverride*: seq[string]
 
-    nocache*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool
+    nocache*, nocomments*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool
 
-    code*, mode*, pluginSourcePath*, sourceFile*: string
+    code*, mode*, pluginSourcePath*: string
 
     onSymbol*: OnSymbol
 
   NimState {.used.} = ref object
     identifiers*: TableRef[string, string]
 
-    constStr*, debugStr*, enumStr*, procStr*, typeStr*: string
+    commentStr*, constStr*, debugStr*, enumStr*, procStr*, typeStr*: string
 
-    debug*: bool
+    gState*: State
 
-    currentHeader*: string
+    currentHeader*, impHeader*, sourceFile*: string
 
     data*: seq[tuple[name, val: string]]
 
 var
-  gStateCT {.compiletime, used.}: State
+  gStateCT {.compiletime, used.} = new(State)
-  gStateRT {.used.}: State
 
 template nBl(s: typed): untyped {.used.} =
   (s.len != 0)
@@ -84,4 +86,4 @@ type CompileMode = enum
 const modeDefault {.used.} = $cpp # TODO: USE this everywhere relevant
 
 when not declared(CIMPORT):
-  export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, gStateRT, nBl, CompileMode, modeDefault
+  export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, NimState, nBl, CompileMode, modeDefault

nimterop/grammar.nim

@@ -7,10 +7,26 @@ import "."/[getters, globals, lisp, treesitter/api]
 type
   Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, nimState: NimState) {.nimcall.}]]
 
-proc genImportC(origName, nimName: string): string =
+proc genImportC(nimState: NimState, origName, nimName: string): string =
-  result = "importc"
   if nimName != origName:
-    result.add &": \"{origName}\"" # used as {.importc: "foo".}
+    result = &"importc: \"{origName}\", header: {nimState.currentHeader}"
+  else:
+    result = nimState.impHeader
+
+proc genPragma(nimState: NimState, pragmas: varargs[string]): string =
+  result = ""
+  for pragma in pragmas.items():
+    if pragma.len != 0:
+      result &= pragma & ", "
+  if result.len != 0:
+    result = " {." & result[0 .. ^2] & ".}"
+
+  result = result.replace(nimState.impHeader & ", cdecl", nimState.impHeader & "C")
+
+proc getComments(nimState: NimState): string =
+  if not nimState.gState.nocomments and nimState.commentStr.len != 0:
+    result = "\n" & nimState.commentStr
+    nimState.commentStr = ""
 
 proc initGrammar(): Grammar =
   # #define X Y
@@ -21,7 +37,7 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# define X Y"
 
       let
@@ -29,10 +45,10 @@ proc initGrammar(): Grammar =
 
       if val.nBl:
         let
-          name = nimState.data[0].val.getIdentifier(nskConst)
+          name = nimState.getIdentifier(nimState.data[0].val, nskConst)
 
-        if name.nBl and nimState.identifiers.addNewIdentifer(name):
+        if name.nBl and nimState.addNewIdentifer(name):
-          nimState.constStr &= &"\n  {name}* = {val}"
+          nimState.constStr &= &"{nimState.getComments()}\n  {name}* = {val}"
   ))
 
   let
@@ -93,7 +109,7 @@ proc initGrammar(): Grammar =
     """
 
   template funcParamCommon(fname, pname, ptyp, pptr, pout, count, i: untyped): untyped =
-    ptyp = nimState.data[i].val.getIdentifier(nskType, fname)
+    ptyp = nimState.getIdentifier(nimState.data[i].val, nskType, fname)
 
     pptr = ""
     while i+1 < nimState.data.len and nimState.data[i+1].name == "pointer_declarator":
@@ -101,7 +117,7 @@ proc initGrammar(): Grammar =
       i += 1
 
     if i+1 < nimState.data.len and nimState.data[i+1].name == "identifier":
-      pname = nimState.data[i+1].val.getIdentifier(nskParam, fname)
+      pname = nimState.getIdentifier(nimState.data[i+1].val, nskParam, fname)
       i += 2
     else:
       pname = "a" & $count
@@ -134,13 +150,14 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# typedef X Y"
 
       var
         i = 0
-        typ = nimState.data[i].val.getIdentifier(nskType)
+        typ = nimState.getIdentifier(nimState.data[i].val, nskType)
         name = ""
+        nname = ""
         tptr = ""
         aptr = ""
 
@@ -155,13 +172,17 @@ proc initGrammar(): Grammar =
             i += 1
 
       if i < nimState.data.len:
-        name = nimState.data[i].val.getIdentifier(nskType)
+        name = nimState.data[i].val
+        nname = nimState.getIdentifier(name, nskType)
         i += 1
 
-      if typ.nBl and name.nBl and nimState.identifiers.addNewIdentifer(name):
+      let
+        pragma = nimState.genPragma(nimState.genImportC(name, nname))
+
+      if typ.nBl and nname.nBl and nimState.addNewIdentifer(nname):
         if i < nimState.data.len and nimState.data[^1].name == "function_declarator":
           var
-            fname = name
+            fname = nname
             pout, pname, ptyp, pptr = ""
             count = 1
 
@@ -175,31 +196,31 @@ proc initGrammar(): Grammar =
             pout = pout[0 .. ^3]
 
           if tptr.len != 0 or typ != "object":
-            nimState.typeStr &= &"\n  {name}* = proc({pout}): {getPtrType(tptr&typ)} {{.nimcall.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}"
           else:
-            nimState.typeStr &= &"\n  {name}* = proc({pout}) {{.nimcall.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = proc({pout}) {{.cdecl.}}"
         else:
           if i < nimState.data.len and nimState.data[i].name in ["identifier", "number_literal"]:
             var
               flen = nimState.data[i].val
             if nimState.data[i].name == "identifier":
-              flen = flen.getIdentifier(nskConst, name)
+              flen = nimState.getIdentifier(flen, nskConst, nname)
 
-            nimState.typeStr &= &"\n  {name}* = {aptr}array[{flen}, {getPtrType(tptr&typ)}]"
+            nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]"
           else:
-            if name == typ:
+            if nname == typ:
-              nimState.typeStr &= &"\n  {name}* = object"
+              nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = object"
             else:
-              nimState.typeStr &= &"\n  {name}* = {getPtrType(tptr&typ)}"
+              nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = {getPtrType(tptr&typ)}"
   ))
 
   proc pDupTypeCommon(nname: string, fend: int, nimState: NimState, isEnum=false) =
-    if nimState.debug:
+    if nimState.gState.debug:
       nimState.debugStr &= "\n#   pDupTypeCommon()"
 
     var
       dname = nimState.data[^1].val
-      ndname = nimState.data[^1].val.getIdentifier(nskType)
+      ndname = nimState.getIdentifier(dname, nskType)
       dptr =
         if fend == 2:
           "ptr "
@@ -208,19 +229,21 @@ proc initGrammar(): Grammar =
 
     if ndname.nBl and ndname != nname:
       if isEnum:
-        if nimState.identifiers.addNewIdentifer(ndname):
+        if nimState.addNewIdentifer(ndname):
-          nimState.enumStr &= &"\ntype {ndname}* = {dptr}{nname}"
+          nimState.enumStr &= &"{nimState.getComments()}\ntype {ndname}* = {dptr}{nname}"
       else:
-        if nimState.identifiers.addNewIdentifer(ndname):
+        if nimState.addNewIdentifer(ndname):
+          let
+            pragma = nimState.genPragma(nimState.genImportc(dname, ndname), "bycopy")
           nimState.typeStr &=
-            &"\n  {ndname}* {{.{genImportC(dname, ndname)}, header: {nimState.currentHeader}, bycopy.}} = {dptr}{nname}"
+            &"{nimState.getComments()}\n  {ndname}*{pragma} = {dptr}{nname}"
 
   proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, nimState: NimState) =
-    if nimState.debug:
+    if nimState.gState.debug:
       nimState.debugStr &= "\n#   pStructCommon"
 
     var
-      nname = name.getIdentifier(nskType)
+      nname = nimState.getIdentifier(name, nskType)
       prefix = ""
       union = ""
 
@@ -246,12 +269,13 @@ proc initGrammar(): Grammar =
                   union = " {.union.}"
               break
 
-    if nname.nBl and nimState.identifiers.addNewIdentifer(nname):
+    if nname.nBl and nimState.addNewIdentifer(nname):
       if nimState.data.len == 1:
-        nimState.typeStr &= &"\n  {nname}* {{.bycopy.}} = object{union}"
+        nimState.typeStr &= &"{nimState.getComments()}\n  {nname}* {{.bycopy.}} = object{union}"
       else:
-        let importCDecl = genImportC(prefix & name, nname)
+        let
-        nimState.typeStr &= &"\n  {nname}* {{.{importCDecl}, header: {nimState.currentHeader}, bycopy.}} = object{union}"
+          pragma = nimState.genPragma(nimState.genImportC(prefix & name, nname), "bycopy")
+        nimState.typeStr &= &"{nimState.getComments()}\n  {nname}*{pragma} = object{union}"
 
       var
         i = fstart
@@ -278,12 +302,12 @@ proc initGrammar(): Grammar =
               aptr &= "ptr "
               i += 1
 
-        fname = nimState.data[i].val.getIdentifier(nskField, nname)
+        fname = nimState.getIdentifier(nimState.data[i].val, nskField, nname)
 
         if i+1 < nimState.data.len-fend and nimState.data[i+1].name in gEnumVals:
           let
             flen = nimState.data[i+1].val.getNimExpression()
-          nimState.typeStr &= &"\n    {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]"
+          nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]"
           i += 2
         elif i+1 < nimState.data.len-fend and nimState.data[i+1].name == "function_declarator":
           var
@@ -304,15 +328,15 @@ proc initGrammar(): Grammar =
           if pout.len != 0 and pout[^2 .. ^1] == ", ":
             pout = pout[0 .. ^3]
           if fptr.len != 0 or ftyp != "object":
-            nimState.typeStr &= &"\n    {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.nimcall.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}"
           else:
-            nimState.typeStr &= &"\n    {fname}*: proc({pout}) {{.nimcall.}}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: proc({pout}) {{.cdecl.}}"
             i += 1
         else:
           if ftyp == "object":
-            nimState.typeStr &= &"\n    {fname}*: pointer"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: pointer"
           else:
-            nimState.typeStr &= &"\n    {fname}*: {getPtrType(fptr&ftyp)}"
+            nimState.typeStr &= &"{nimState.getComments()}\n    {fname}*: {getPtrType(fptr&ftyp)}"
           i += 1
 
       if node.tsNodeType() == "type_definition" and
@@ -366,7 +390,7 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# struct X {}"
 
       pStructCommon(ast, node, nimState.data[0].val, 1, 1, nimState)
@@ -389,7 +413,7 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# typedef struct X {}"
 
       var
@@ -410,17 +434,17 @@ proc initGrammar(): Grammar =
   ))
 
   proc pEnumCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, nimState: NimState) =
-    if nimState.debug:
+    if nimState.gState.debug:
       nimState.debugStr &= "\n#   pEnumCommon()"
 
     let nname =
       if name.len == 0:
-        getUniqueIdentifier(nimState.identifiers, "Enum")
+        getUniqueIdentifier(nimState, "Enum")
       else:
-        name.getIdentifier(nskType)
+        nimState.getIdentifier(name, nskType)
 
-    if nname.nBl and nimState.identifiers.addNewIdentifer(nname):
+    if nname.nBl and nimState.addNewIdentifer(nname):
-      nimState.enumStr &= &"\ndefineEnum({nname})"
+      nimState.enumStr &= &"{nimState.getComments()}\ndefineEnum({nname})"
 
       var
         i = fstart
@@ -431,20 +455,20 @@ proc initGrammar(): Grammar =
           continue
 
         let
-          fname = nimState.data[i].val.getIdentifier(nskEnumField)
+          fname = nimState.getIdentifier(nimState.data[i].val, nskEnumField)
 
         if i+1 < nimState.data.len-fend and
           nimState.data[i+1].name in gEnumVals:
-          if fname.nBl and nimState.identifiers.addNewIdentifer(fname):
+          if fname.nBl and nimState.addNewIdentifer(fname):
-            nimState.constStr &= &"\n  {fname}* = ({nimState.data[i+1].val.getNimExpression()}).{nname}"
+            nimState.constStr &= &"{nimState.getComments()}\n  {fname}* = ({nimState.data[i+1].val.getNimExpression()}).{nname}"
           try:
             count = nimState.data[i+1].val.parseInt() + 1
           except:
             count += 1
           i += 2
         else:
-          if fname.nBl and nimState.identifiers.addNewIdentifer(fname):
+          if fname.nBl and nimState.addNewIdentifer(fname):
-            nimState.constStr &= &"\n  {fname}* = {count}.{nname}"
+            nimState.constStr &= &"{nimState.getComments()}\n  {fname}* = {count}.{nname}"
           i += 1
           count += 1
 
@@ -465,7 +489,7 @@ proc initGrammar(): Grammar =
    )
   """ % gEnumVals.join("|"),
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# enum X {}"
 
       var
@@ -493,7 +517,7 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# typedef enum {}"
 
       var
@@ -526,7 +550,7 @@ proc initGrammar(): Grammar =
    )
   """,
     proc (ast: ref Ast, node: TSNode, nimState: NimState) =
-      if nimState.debug:
+      if nimState.gState.debug:
         nimState.debugStr &= "\n# typ function"
 
       var
@@ -545,7 +569,7 @@ proc initGrammar(): Grammar =
 
         var
           fname = nimState.data[i].val
-          fnname = fname.getIdentifier(nskProc)
+          fnname = nimState.getIdentifier(fname, nskProc)
           pout, pname, ptyp, pptr = ""
           count = 1
 
@@ -559,12 +583,31 @@ proc initGrammar(): Grammar =
         if pout.len != 0 and pout[^2 .. ^1] == ", ":
           pout = pout[0 .. ^3]
 
-        if fnname.nBl and nimState.identifiers.addNewIdentifer(fnname):
+        if fnname.nBl and nimState.addNewIdentifer(fnname):
-          let ftyp = nimState.data[0].val.getIdentifier(nskType, fnname)
+          let
+            ftyp = nimState.getIdentifier(nimState.data[0].val, nskType, fnname)
+            pragma = nimState.genPragma(nimState.genImportC(fname, fnname), "cdecl")
+
           if fptr.len != 0 or ftyp != "object":
-            nimState.procStr &= &"\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)} {{.{genImportC(fname, fnname)}, header: {nimState.currentHeader}.}}"
+            nimState.procStr &= &"{nimState.getComments()}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}"
           else:
-            nimState.procStr &= &"\nproc {fnname}*({pout}) {{.{genImportC(fname, fnname)}, header: {nimState.currentHeader}.}}"
+            nimState.procStr &= &"{nimState.getComments()}\nproc {fnname}*({pout}){pragma}"
+  ))
+
+  # // comment
+  result.add((&"""
+   (comment
+   )
+  """,
+    proc (ast: ref Ast, node: TSNode, nimState: NimState) =
+      let
+        cmt = $nimState.getNodeVal(node)
+
+      for line in cmt.splitLines():
+        let
+          line = line.multiReplace([("//", ""), ("/*", ""), ("*/", "")])
+
+        nimState.commentStr &= &"\n#{line.strip(leading=false)}"
   ))
 
 proc initRegex(ast: ref Ast) =

nimterop/lisp.nim

@@ -30,7 +30,6 @@ proc readFromTokens(): ref Ast =
   if gTokens[idx] == "(":
     if gTokens.len - idx < 2:
       doAssert false, "Corrupt AST " & $(gTokensLen: gTokens.len, idx: idx)
-    if gTokens[idx+1] != "comment":
     result = new(Ast)
     (result.name, result.kind, result.recursive) = gTokens[idx+1].getNameKind()
     result.children = @[]

nimterop/toast.nim

@@ -4,7 +4,7 @@ import "."/treesitter/[api, c, cpp]
 
 import "."/[ast, globals, getters, grammar]
 
-proc printLisp(root: TSNode) =
+proc printLisp(gState: State, root: TSNode) =
   var
     node = root
     nextnode: TSNode
@@ -12,21 +12,21 @@ proc printLisp(root: TSNode) =
 
   while true:
     if not node.tsNodeIsNull() and depth > -1:
-      if gStateRT.pretty:
+      if gState.pretty:
         stdout.write spaces(depth)
       let
-        (line, col) = node.getLineCol()
+        (line, col) = gState.getLineCol(node)
       stdout.write &"({$node.tsNodeType()} {line} {col} {node.tsNodeEndByte() - node.tsNodeStartByte()}"
     else:
       break
 
     if node.tsNodeNamedChildCount() != 0:
-      if gStateRT.pretty:
+      if gState.pretty:
         echo ""
       nextnode = node.tsNodeNamedChild(0)
       depth += 1
     else:
-      if gStateRT.pretty:
+      if gState.pretty:
         echo ")"
       else:
         stdout.write ")"
@@ -38,7 +38,7 @@ proc printLisp(root: TSNode) =
         depth -= 1
         if depth == -1:
           break
-        if gStateRT.pretty:
+        if gState.pretty:
           echo spaces(depth) & ")"
         else:
           stdout.write ")"
@@ -53,7 +53,7 @@ proc printLisp(root: TSNode) =
     if node == root:
       break
 
-proc process(path: string, astTable: AstTable) =
+proc process(gState: State, path: string, astTable: AstTable) =
   doAssert existsFile(path), "Invalid path " & path
 
   var
@@ -63,110 +63,111 @@ proc process(path: string, astTable: AstTable) =
   defer:
     parser.tsParserDelete()
 
-  gStateRT.sourceFile = path
+  if gState.mode.len == 0:
-
-  if gStateRT.mode.len == 0:
     if ext in [".h", ".c"]:
-      gStateRT.mode = "c"
+      gState.mode = "c"
     elif ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]:
-      gStateRT.mode = "cpp"
+      gState.mode = "cpp"
 
-  if gStateRT.preprocess:
+  if gState.preprocess:
-    gStateRT.code = getPreprocessor(path)
+    gState.code = gState.getPreprocessor(path)
   else:
-    gStateRT.code = readFile(path)
+    gState.code = readFile(path)
 
-  doAssert gStateRT.code.len != 0, "Empty file or preprocessor error"
+  doAssert gState.code.len != 0, "Empty file or preprocessor error"
 
-  if gStateRT.mode == "c":
+  if gState.mode == "c":
     doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser"
-  elif gStateRT.mode == "cpp":
+  elif gState.mode == "cpp":
     doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
   else:
-    doAssert false, "Invalid parser " & gStateRT.mode
+    doAssert false, "Invalid parser " & gState.mode
 
   var
-    tree = parser.tsParserParseString(nil, gStateRT.code.cstring, gStateRT.code.len.uint32)
+    tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32)
     root = tree.tsTreeRootNode()
 
   defer:
     tree.tsTreeDelete()
 
-  if gStateRT.past:
+  if gState.past:
-    printLisp(root)
+    gState.printLisp(root)
-  elif gStateRT.pnim:
+  elif gState.pnim:
-    printNim(path, root, astTable)
+    gState.printNim(path, root, astTable)
-  elif gStateRT.preprocess:
+  elif gState.preprocess:
-    echo gStateRT.code
+    echo gState.code
 
 proc main(
-    mode = modeDefault,
+    preprocess = false,
     past = false,
     pnim = false,
-    pretty = true,
-    preprocess = false,
-    pgrammar = false,
     recurse = false,
-    debug = false,
+    nocomments = false,
     defines: seq[string] = @[],
     includeDirs: seq[string] = @[],
     symOverride: seq[string] = @[],
     pluginSourcePath: string = "",
-    source: seq[string],
+    debug = false,
+    mode = modeDefault,
+    pgrammar = false,
+    source: seq[string]
   ) =
 
-  gStateRT = State(
+  var gState = State(
-    mode: mode,
+    preprocess: preprocess,
     past: past,
     pnim: pnim,
-    pretty: pretty,
-    preprocess: preprocess,
     recurse: recurse,
-    debug: debug,
+    nocomments: nocomments,
     defines: defines,
     includeDirs: includeDirs,
     symOverride: symOverride,
-    pluginSourcePath: pluginSourcePath
+    pluginSourcePath: pluginSourcePath,
+    debug: debug,
+    mode: mode,
+    pretty: true
   )
 
-  gStateRT.symOverride = gStateRT.symOverride.getSplitComma()
+  gState.symOverride = gState.symOverride.getSplitComma()
 
   if pluginSourcePath.nBl:
-    loadPlugin(pluginSourcePath)
+    gState.loadPlugin(pluginSourcePath)
 
   let
     astTable = parseGrammar()
   if pgrammar:
     astTable.printGrammar()
   elif source.len != 0:
-    if gStateRT.pnim:
+    if gState.pnim:
       printNimHeader()
     for src in source:
-      process(src, astTable)
+      gState.process(src, astTable)
 
 when isMainModule:
   import cligen
   dispatch(main, help = {
+    "preprocess": "run preprocessor on header",
     "past": "print AST output",
-    "mode": "language; see CompileMode", # TODO: auto-generate valid choices
     "pnim": "print Nim output",
+    "recurse": "process #include files",
+    "nocomments": "exclude top-level comments from output",
     "defines": "definitions to pass to preprocessor",
     "includeDirs": "include directory to pass to preprocessor",
     "symOverride": "skip generating specified symbols",
     "pluginSourcePath": "Nim file to build and load as a plugin",
-    "preprocess": "run preprocessor on header",
-    "pgrammar": "print grammar",
-    "recurse": "process #include files",
     "debug": "enable debug output",
-    "source" : "C/C++ source/header",
+    "mode": "language parser: c or cpp",
+    "pgrammar": "print grammar",
+    "source" : "C/C++ source/header"
   }, short = {
+    "preprocess": 'p',
     "past": 'a',
     "pnim": 'n',
+    "recurse": 'r',
+    "nocomments": 'c',
     "defines": 'D',
     "includeDirs": 'I',
     "symOverride": 'O',
-    "preprocess": 'p',
-    "recurse": 'r',
     "debug": 'd',
     "pgrammar": 'g'
   })

tests/include/test.h

@@ -27,6 +27,7 @@ extern "C" {
 
 typedef uint8_t PRIMTYPE;
 typedef PRIMTYPE CUSTTYPE;
+typedef CUSTTYPE _CCUSTTYPE_;
 
 struct STRUCT0;
 
@@ -72,6 +73,11 @@ enum ENUM5 {
 	enum15 = (1 << (1 & 1))
 };
 
+enum ENUM7 {
+  enum17 = '\0',
+  enum18 = 'A'
+};
+
 typedef void * VOIDPTR;
 typedef int * INTPTR;
 

tests/tmath.nim

@@ -6,6 +6,11 @@ type
   mingw_ldbl_type_t = object
   mingw_dbl_type_t = object
 
+when defined(windows):
+  cOverride:
+    type
+      complex = object
+
 static:
   cDebug()
   cDisableCaching()

tests/tnimterop_c.nim

@@ -40,6 +40,7 @@ block:
 var
   pt: PRIMTYPE
   ct: CUSTTYPE
+  cct: CCUSTTYPE
 
   s0: STRUCT0
   s1: STRUCT1
@@ -64,6 +65,7 @@ var
 
 pt = 3
 ct = 4
+cct = 5
 
 s1.field1 = 5
 s2.field1 = 6
@@ -125,6 +127,8 @@ check e4 == enum11
 check enum13 == 4
 check enum14 == 9
 check enum15 == 2
+check enum17 == '\0'.ENUM7
+check enum18 == 'A'.ENUM7
 
 # Issue #58
 multiline1()