Remove unnecessary proc

nimterop/ast2.nim

@@ -1,10 +1,12 @@
 import macros, os, sequtils, sets, strformat, strutils, tables, times
 
-import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, parser, renderer]
+import options as opts
+
+import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer]
 
 import "."/treesitter/api
 
-import "."/[globals, getters, exprparser]
+import "."/[globals, getters, exprparser, comphelp, tshelp]
 
 proc getPtrType*(str: string): string =
   result = case str:
@@ -17,24 +19,6 @@ proc getPtrType*(str: string): string =
     else:
       str
 
-proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) =
-  # Raise exception in parseString() instead of exiting for errors
-  if msg < warnMin:
-    raise newException(Exception, msgKindToString(msg))
-
-proc parseString(gState: State, str: string): PNode =
-  # Parse a string into Nim AST - use custom error handler that raises
-  # an exception rather than exiting on failure
-  try:
-    result = parseString(
-      str, gState.identCache, gState.config, errorHandler = handleError
-    )
-  except:
-    decho getCurrentExceptionMsg()
-
-proc getLit*(nimState: NimState, str: string, expression = false): PNode =
-  result = nimState.codeToNode(str)
-
 proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode =
   # Check if symbol `origname` of `kind` and `origname` has any cOverride defined
   # and use that if present
@@ -57,6 +41,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS
       result = pnode[0][0]
   else:
     gecho &"\n# $1'{origname}' skipped" % skind
+    gState.skippedSyms.incl origname
     if gState.debug:
       gState.skipStr &= &"\n{gState.getNodeVal(node)}"
 
@@ -115,14 +100,37 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode =
         fval
       else:
         gState.getNodeVal(node[1])
-    valident =
-      gState.getLit(val)
+
+  var valident = newNode(nkNone)
+
+  withCodeAst(val, gState.mode):
+    # This section is a hack for determining that the first
+    # node is a type, which shouldn't be accepted by a const
+    # def section. Need to replace this with some other mechanism
+    # to handle type aliases
+    var maybeTyNode: TSNode
+    # Take the very first node, which may be 2 levels
+    # down if there is an error node
+    if root.len > 0 and root[0].getName() == "ERROR":
+      maybeTyNode = root[0][0]
+    elif root.len > 0:
+      maybeTyNode = root[0]
+
+    if not maybeTyNode.isNil:
+      let name = maybeTyNode.getName()
+      case name
+      of "type_descriptor", "sized_type_specifier":
+        discard
+      else:
+        # Can't do gState.parseCExpression(root) here for some reason?
+        # get a SEGFAULT if we use root
+        valident = gState.parseCExpression(val)
 
   if name.Bl:
     # Name skipped or overridden since blank
-    result = nimState.getOverrideOrSkip(node, origname, nskConst)
-  elif valident.kind != nkNilLit:
-    if nimState.addNewIdentifer(name):
+    result = gState.getOverrideOrSkip(node, origname, nskConst)
+  elif valident.kind != nkNone:
+    if gState.addNewIdentifer(name):
       # const X* = Y
       #
       # nkConstDef(
@@ -145,6 +153,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode =
       gecho &"# const '{origname}' is duplicate, skipped"
   else:
     gecho &"# const '{origname}' has invalid value '{val}'"
+    gState.skippedSyms.incl origname
 
 proc addConst(gState: State, node: TSNode) =
   # Add a const to the AST
@@ -682,6 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
         let
           fdecl = node[i].anyChildInTree("field_declaration_list")
           edecl = node[i].anyChildInTree("enumerator_list")
+          commentNodes = node[i].getNextCommentNodes()
 
           # `tname` is name of nested struct / union / enum just
           # added, passed on as type name for field in `newIdentDefs()`
@@ -707,6 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
         # Add nkIdentDefs for each field
         for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true):
           if not field.isNil:
+            field.comment = gState.getCommentsStr(commentNodes)
             result.add field
 
 proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) =
@@ -716,6 +727,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
   # If `fname` is set, use it as the name when creating new PNode
   # If `istype` is set, this is a typedef, else struct/union
   decho("addTypeObject()")
+  let commentNodes = node.tsNodeParent().getPrevCommentNodes()
+
   let
     # Object has fields or not
     fdlist = node.anyChildInTree("field_declaration_list")
@@ -828,6 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
         gState.addPragma(node, typeDef[0][1], pragmas)
 
     # nkTypeSection.add
+    typeDef.comment = gState.getCommentsStr(commentNodes)
     gState.typeSection.add typeDef
 
     gState.printDebug(typeDef)
@@ -839,6 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
       # Current node has fields
       let
         origname = gState.getNodeVal(node.getAtom())
+        commentNodes = node.getNextCommentNodes()
 
         # Fix issue #185
         name =
@@ -850,6 +865,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
       if name.nBl and gState.identifierNodes.hasKey(name):
         let
           def = gState.identifierNodes[name]
+        def.comment = gState.getCommentsStr(commentNodes)
+
         # Duplicate nkTypeDef for `name` with empty fields
         if def.kind == nkTypeDef and def.len == 3 and
           def[2].kind == nkObjectTy and def[2].len == 3 and
@@ -881,6 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) =
   decho("addTypeTyped()")
   let
     start = getStartAtom(node)
+    commentNodes = node.getPrevCommentNodes()
   for i in start+1+offset ..< node.len:
     # Add a type of a specific type
     let
@@ -888,6 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) =
       typeDef = gState.newXIdent(node[i], istype = true)
 
     if not typeDef.isNil:
+      typeDef.comment = gState.getCommentsStr(commentNodes)
       let
         name = typeDef.getIdentName()
 
@@ -977,8 +996,8 @@ proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNo
       # type name[X] => array[X, type]
       let
         # Size of array could be a Nim expression
-        size = gState.getLit(gState.getNodeVal(cnode[1]), expression = true)
-      if size.kind != nkNilLit:
+        size = gState.parseCExpression(gState.getNodeVal(cnode[1]))
+      if size.kind != nkNone:
         result = gState.newArrayTree(cnode, result, size)
         cnode = cnode[0]
     elif cnode.len == 1:
@@ -1377,18 +1396,33 @@ proc addEnum(gState: State, node: TSNode) =
         gState.typeSection.add eoverride
     elif gState.addNewIdentifer(name):
       # Add enum definition and helpers
-      gState.enumSection.add gState.parseString(&"defineEnum({name})")
+      let defineNode = gState.parseString(&"defineEnum({name})")
+      # nkStmtList(
+      #  nkCall(
+      #   nkIdent("defineEnum"),
+      #   nkIdent(name) <- set the comment here
+      #  )
+      # )
+      defineNode[0][1].comment = gState.getCommentsStr(node.getPrevCommentNodes())
+      gState.enumSection.add defineNode
 
       # Create const for fields
       var
         fnames: HashSet[string]
+        # Hold all of field information so that we can add all of them
+        # after the const identifiers has been updated
+        fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]]
       for i in 0 .. enumlist.len - 1:
         let
           en = enumlist[i]
         if en.getName() == "comment":
           continue
+
         let
-          fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField)
+          atom = en.getAtom()
+          commentNodes = en.getNextCommentNodes()
+          fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField)
+
         if fname.nBl:
           var
             fval = ""
@@ -1400,25 +1434,32 @@ proc addEnum(gState: State, node: TSNode) =
             fval = &"({prev} + 1).{name}"
 
           if en.len > 1 and en[1].getName() in gEnumVals:
-            # Explicit value
-            fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name
-
-          # Cannot use newConstDef() since parseString(fval) adds backticks to and/or
-          gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0]
+            fieldDeclarations.add((fname, "", some(en[1]), commentNodes))
+          else:
+            fieldDeclarations.add((fname, fval, none(TSNode), commentNodes))
 
           fnames.incl fname
-
           prev = fname
 
       # Add fields to list of consts after processing enum so that we don't cast
       # enum field to itself
       gState.constIdentifiers.incl fnames
 
+      # parseCExpression requires all const identifiers to be present for the enum
+      for (fname, fval, cexprNode, commentNodes) in fieldDeclarations:
+        var fval = fval
+        if cexprNode.isSome:
+          fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name
+        # Cannot use newConstDef() since parseString(fval) adds backticks to and/or
+        let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0]
+        constNode.comment = gState.getCommentsStr(commentNodes)
+        gState.constSection.add constNode
+
       # Add other names
       if node.getName() == "type_definition" and node.len > 1:
         gState.addTypeTyped(node, ftname = name, offset = offset)
 
-proc addProcVar(gState: State, node, rnode: TSNode) =
+proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) =
   # Add a proc variable
   decho("addProcVar()")
   let
@@ -1471,12 +1512,13 @@ proc addProcVar(gState: State, node, rnode: TSNode) =
     #  nkEmpty()
     # )
 
+    identDefs.comment = gState.getCommentsStr(commentNodes)
     # nkVarSection.add
     gState.varSection.add identDefs
 
     gState.printDebug(identDefs)
 
-proc addProc(gState: State, node, rnode: TSNode) =
+proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) =
   # Add a proc
   #
   # `node` is the `nth` child of (declaration)
@@ -1582,6 +1624,8 @@ proc addProc(gState: State, node, rnode: TSNode) =
     procDef.add newNode(nkEmpty)
     procDef.add newNode(nkEmpty)
 
+    procDef.comment = gState.getCommentsStr(commentNodes)
+
     # nkProcSection.add
     gState.procSection.add procDef
 
@@ -1594,16 +1638,17 @@ proc addDecl(gState: State, node: TSNode) =
 
   let
     start = getStartAtom(node)
+    commentNodes = node.getPrevCommentNodes()
 
   for i in start+1 ..< node.len:
     if not node[i].firstChildInTree("function_declarator").isNil:
       # Proc declaration - var or actual proc
       if node[i].getAtom().getPxName(1) == "pointer_declarator":
         # proc var
-        gState.addProcVar(node[i], node[start])
+        gState.addProcVar(node[i], node[start], commentNodes)
       else:
         # proc
-        gState.addProc(node[i], node[start])
+        gState.addProc(node[i], node[start], commentNodes)
     else:
       # Regular var
       discard
@@ -1615,11 +1660,14 @@ proc addDef(gState: State, node: TSNode) =
   # and will fail at link time
   decho("addDef()")
   gState.printDebug(node)
+
   let
     start = getStartAtom(node)
+    commentNodes = node.getPrevCommentNodes()
+
   if node[start+1].getName() == "function_declarator":
     if gState.isIncludeHeader():
-      gState.addProc(node[start+1], node[start])
+      gState.addProc(node[start+1], node[start], commentNodes)
     else:
       gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" %
         gState.getNodeVal(node[start+1].getAtom())

nimterop/comphelp.nim

@@ -0,0 +1,18 @@
+import compiler/[ast, lineinfos, msgs, options, parser, renderer]
+
+import "."/[globals, getters]
+
+proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) =
+  # Raise exception in parseString() instead of exiting for errors
+  if msg < warnMin:
+    raise newException(Exception, msgKindToString(msg))
+
+proc parseString*(gState: State, str: string): PNode =
+  # Parse a string into Nim AST - use custom error handler that raises
+  # an exception rather than exiting on failure
+  try:
+    result = parseString(
+      str, gState.identCache, gState.config, errorHandler = handleError
+    )
+  except:
+    decho getCurrentExceptionMsg()

nimterop/exprparser.nim

@@ -1,4 +1,4 @@
-import strformat, strutils, macros
+import strformat, strutils, macros, sets
 
 import regex
 
@@ -6,70 +6,133 @@ import compiler/[ast, renderer]
 
 import "."/treesitter/[api, c, cpp]
 
-import "."/[globals, getters]
+import "."/[globals, getters, comphelp, tshelp]
+
+# This version of exprparser should be able to handle:
+#
+# All integers + integer like expressions (hex, octal, suffixes)
+# All floating point expressions (except for C++'s hex floating point stuff)
+# Strings and character literals, including C's escape characters (not sure if this is the same as C++'s escape characters or not)
+# Math operators (+, -, /, *)
+# Some Unary operators (-, !, ~). ++, --, and & are yet to be implemented
+# Any identifiers
+# C type descriptors (int, char, etc)
+# Boolean values (true, false)
+# Shift expressions (containing anything in this list)
+# Cast expressions (containing anything in this list)
+# Math expressions (containing anything in this list)
+# Sizeof expressions (containing anything in this list)
+# Cast expressions (containing anything in this list)
+# Parentheses expressions (containing anything in this list)
+# Expressions containing other expressions
+#
+# In addition to the above, it should also handle most type coercions, except
+# for where Nim can't (such as uint + -int)
 
 type
-  ExprParser* = ref object
-    state*: NimState
-    code*: string
-
   ExprParseError* = object of CatchableError
 
-proc newExprParser*(state: NimState, code: string): ExprParser =
-  ExprParser(state: state, code: code)
+template val(node: TSNode): string =
+  gState.currentExpr.getNodeVal(node)
 
-template techo(msg: varargs[string, `$`]) =
-  if exprParser.state.gState.debug:
-    let nimState {.inject.} = exprParser.state
-    necho "# " & join(msg, "").replace("\n", "\n# ")
+proc printDebugExpr*(gState: State, node: TSNode) =
+  if gState.debug:
+    gecho ("Input => " & node.val).getCommented()
+    gecho gState.currentExpr.printLisp(node).getCommented()
 
-template val*(node: TSNode): string =
-  exprParser.code.getNodeVal(node)
+proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode =
+  ## Gets a cPlugin transformed identifier from `identName`
+  ##
+  ## Returns PNode(nkNone) if the identifier is blank
+  result = newNode(nkNone)
+  if identName notin gState.skippedSyms:
+    var ident = identName
+    if ident != "_":
+      # Process the identifier through cPlugin
+      ident = gState.getIdentifier(ident, kind, parent)
+    if kind == nskType:
+      result = gState.getIdent(ident)
+    elif ident.nBl and ident in gState.constIdentifiers:
+      if gState.currentTyCastName.nBl:
+        ident = ident & "." & gState.currentTyCastName
+      result = gState.getIdent(ident)
 
-proc mode*(exprParser: ExprParser): string =
-  exprParser.state.gState.mode
+proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode =
+  ## Gets a cPlugin transformed identifier from `identName`
+  ##
+  ## Returns PNode(nkNone) if the identifier is blank
+  gState.getExprIdent(node.val, kind, parent)
 
-template withCodeAst(exprParser: ExprParser, body: untyped): untyped =
-  var parser = tsParserNew()
-  defer:
-    parser.tsParserDelete()
+proc parseChar(charStr: string): uint8 {.inline.} =
+  ## Parses a character literal out of a string. This is needed
+  ## because treesitter gives unescaped characters when parsing
+  ## strings.
+  if charStr.len == 1:
+    return charStr[0].uint8
 
-  doAssert exprParser.code.nBl, "Empty code"
-  if exprParser.mode == "c":
-    doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser"
-  elif exprParser.mode == "cpp":
-    doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
-  else:
-    doAssert false, &"Invalid parser {exprParser.mode}"
+  # Handle octal, hex, unicode?
+  if charStr.startsWith("\\x"):
+    result = parseHexInt(charStr.replace("\\x", "0x")).uint8
+  elif charStr.len == 4: # Octal
+    result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8
 
-  var
-    tree = parser.tsParserParseString(nil, exprParser.code.cstring, exprParser.code.len.uint32)
-    root {.inject.} = tree.tsTreeRootNode()
+  if result == 0:
+    case charStr
+    of "\\0":
+      result = ord('\0')
+    of "\\a":
+      result = 0x07
+    of "\\b":
+      result = 0x08
+    of "\\e":
+      result = 0x1B
+    of "\\f":
+      result = 0x0C
+    of "\\n":
+      result = '\n'.uint8
+    of "\\r":
+      result = 0x0D
+    of "\\t":
+      result = 0x09
+    of "\\v":
+      result = 0x0B
+    of "\\\\":
+      result = 0x5C
+    of "\\'":
+      result = '\''.uint8
+    of "\\\"":
+      result = '\"'.uint8
+    of "\\?":
+      result = 0x3F
+    else:
+      discard
 
-  body
+  if result > uint8.high:
+    result = uint8.high
 
-  defer:
-    tree.tsTreeDelete()
+proc getCharLit(charStr: string): PNode {.inline.} =
+  ## Convert a character string into a proper Nim char lit node
+  result = newNode(nkCharLit)
+  result.intVal = parseChar(charStr).int64
 
+proc getFloatNode(number, suffix: string): PNode {.inline.} =
+  ## Get a Nim float node from a C float expression + suffix
+  let floatSuffix = number[number.len-1]
+  try:
+    case floatSuffix
+    of 'l', 'L':
+      # TODO: handle long double (128 bits)
+      # result = newNode(nkFloat128Lit)
+      result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
+    of 'f', 'F':
+      result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
+    else:
+      result = newFloatNode(nkFloatLit, parseFloat(number))
+  except ValueError:
+    raise newException(ExprParseError, &"Could not parse float value \"{number}\".")
 
-proc getNumNode(number, suffix: string): PNode {.inline.} =
-  result = newNode(nkNilLit)
-  if number.contains("."):
-    let floatSuffix = number[result.len-1]
-    try:
-      case floatSuffix
-      of 'l', 'L':
-        # TODO: handle long double (128 bits)
-        # result = newNode(nkFloat128Lit)
-        result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
-      of 'f', 'F':
-        result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
-      else:
-        result = newFloatNode(nkFloatLit, parseFloat(number[0 ..< number.len - 1]))
-      return
-    except ValueError:
-      raise newException(ExprParseError, &"Could not parse float value \"{number}\".")
-
+proc getIntNode(number, suffix: string): PNode {.inline.} =
+  ## Get a Nim int node from a C integer expression + suffix
   case suffix
   of "u", "U":
     result = newNode(nkUintLit)
@@ -84,6 +147,8 @@ proc getNumNode(number, suffix: string): PNode {.inline.} =
   else:
     result = newNode(nkIntLit)
 
+  # I realize these regex are wasteful on performance, but
+  # couldn't come up with a better idea.
   if number.contains(re"0[xX]"):
     result.intVal = parseHexInt(number)
     result.flags = {nfBase16}
@@ -96,12 +161,20 @@ proc getNumNode(number, suffix: string): PNode {.inline.} =
   else:
     result.intVal = parseInt(number)
 
-proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkNilLit)
+proc getNumNode(number, suffix: string): PNode {.inline.} =
+  ## Convert a C number to a Nim number PNode
+  if number.contains("."):
+    getFloatNode(number, suffix)
+  else:
+    getIntNode(number, suffix)
+
+proc processNumberLiteral(gState: State, node: TSNode): PNode =
+  ## Parse a number literal from a TSNode. Can be a float, hex, long, etc
+  result = newNode(nkNone)
   let nodeVal = node.val
 
   var match: RegexMatch
-  const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.?\d*[fFlL]?|\d*\.?\d+[fFlL]?|\d+)([ulUL]*)"
+  const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)"
   let found = nodeVal.find(reg, match)
   if found:
     let
@@ -111,176 +184,404 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode =
 
     result = getNumNode(number, suffix)
 
-    if result.kind != nkNilLit and prefix == "-":
+    if result.kind != nkNone and prefix == "-":
       result = nkPrefix.newTree(
-        exprParser.state.getIdent("-"),
+        gState.getIdent("-"),
         result
       )
   else:
     raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"")
 
-proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkCharLit)
-  result.intVal = node.val[1].int64
+proc processCharacterLiteral(gState: State, node: TSNode): PNode =
+  # Input => 'G'
+  #
+  # (char_literal 1 1 3 "'G'")
+  #
+  # Output => 'G'
+  #
+  # nkCharLit("G")
+  let val = node.val
+  result = getCharLit(val[1 ..< val.len - 1])
 
-proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode =
-  let nodeVal = node.val
-  result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1])
+proc processStringLiteral(gState: State, node: TSNode): PNode =
+  # Input => "\n\rfoobar\0\'"
+  #
+  # (string_literal 1 1 16 ""\n\rfoobar\0\'""
+  #  (escape_sequence 1 2 2 "\n")
+  #  (escape_sequence 1 4 2 "\r")
+  #  (escape_sequence 1 12 2 "\0")
+  #  (escape_sequence 1 14 2 "\'")
+  # )
+  #
+  # Output => "\n\cfoobar\x00\'"
+  #
+  # nkStrLit("\x0A\x0Dfoobar\x00\'")
+  let
+    nodeVal = node.val
+    strVal = nodeVal[1 ..< nodeVal.len - 1]
 
-proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode
+  const
+    str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])"
+    reg = re(str)
 
-proc processShiftExpression*(exprParser: ExprParser, node: TSNode): PNode =
+  # Convert the c string escape sequences/etc to Nim chars
+  var nimStr = newStringOfCap(nodeVal.len)
+  for m in strVal.findAll(reg):
+    nimStr.add(parseChar(strVal[m.group(0)[0]]).chr)
+
+  result = newStrNode(nkStrLit, nimStr)
+
+proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode
+
+proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  # Input => (a + b)
+  #
+  # (parenthesized_expression 1 1 7
+  #  (math_expression 1 2 5
+  #   (identifier 1 2 1 "a")
+  #   (identifier 1 6 1 "b")
+  #  )
+  # )
+  #
+  # Output => (typeof(a)(a + typeof(a)(b)))
+  #
+  # nkPar(
+  #  nkCall(
+  #   nkCall(
+  #    nkIdent("typeof"),
+  #    nkIdent("a")
+  #   ),
+  #   nkInfix(
+  #    nkIdent("+"),
+  #    nkIdent("a"),
+  #    nkCall(
+  #     nkCall(
+  #      nkIdent("typeof"),
+  #      nkIdent("a")
+  #     ),
+  #     nkIdent("b")
+  #    )
+  #   )
+  #  )
+  # )
+  result = newNode(nkPar)
+  for i in 0 ..< node.len():
+    result.add(gState.processTSNode(node[i], typeofNode))
+
+proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  # Input => (int)a
+  #
+  # (cast_expression 1 1 6 "(int)a"
+  #  (type_descriptor 1 2 3 "int"
+  #   (primitive_type 1 2 3 "int")
+  #  )
+  #  (identifier 1 6 1 "a")
+  # )
+  #
+  # Output => cast[cint](a)
+  #
+  # nkCast(
+  #  nkIdent("cint"),
+  #  nkIdent("a")
+  # )
+  result = nkCast.newTree(
+    gState.processTSNode(node[0], typeofNode),
+    gState.processTSNode(node[1], typeofNode)
+  )
+
+proc getNimUnarySym(csymbol: string): string =
+  ## Get the Nim equivalent of a unary C symbol
+  ##
+  ## TODO: Add ++, --,
+  case csymbol
+  of "+", "-":
+    result = csymbol
+  of "~", "!":
+    result = "not"
+  else:
+    raise newException(ExprParseError, &"Unsupported unary symbol \"{csymbol}\"")
+
+proc getNimBinarySym(csymbol: string): string =
+  case csymbol
+  of "|", "||":
+    result = "or"
+  of "&", "&&":
+    result = "and"
+  of "^":
+    result = "xor"
+  of "==", "!=",
+     "+", "-", "/", "*",
+     ">", "<", ">=", "<=":
+    result = csymbol
+  of "%":
+    result = "mod"
+  of "<<":
+    result = "shl"
+  of ">>":
+    result = "shr"
+  else:
+    raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"")
+
+proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  # Node has left and right children ie: (2 + 7)
+  #
+  # Input => a == b
+  #
+  # (equality_expression 1 1 6
+  #  (identifier 1 1 1 "a")
+  #  (identifier 1 6 1 "b")
+  # )
+  #
+  # Output => a == typeof(a)(b)
+  #
+  # nkInfix(
+  #  nkIdent("=="),
+  #  nkIdent("a"),
+  #  nkCall(
+  #   nkCall(
+  #    nkIdent("typeof"),
+  #    nkIdent("a")
+  #   ),
+  #   nkIdent("b")
+  #  )
+  # )
   result = newNode(nkInfix)
+
   let
     left = node[0]
     right = node[1]
-  var shiftSym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip()
+    binarySym = node.tsNodeChild(1).val.strip()
+    nimSym = getNimBinarySym(binarySym)
 
-  case shiftSym
-  of "<<":
-    result.add exprParser.state.getIdent("shl")
-  of ">>":
-    result.add exprParser.state.getIdent("shr")
-  else:
-    raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"")
+  result.add gState.getIdent(nimSym)
+  let leftNode = gState.processTSNode(left, typeofNode)
 
-  let
-    leftNode = exprParser.processTSNode(left)
-    rightNode = exprParser.processTSNode(right)
+  if typeofNode.isNil:
+    typeofNode = nkCall.newTree(
+      gState.getIdent("typeof"),
+      leftNode
+    )
+
+  let rightNode = gState.processTSNode(right, typeofNode)
 
   result.add leftNode
-  result.add nkCast.newTree(
-    nkCall.newTree(
-      exprParser.state.getIdent("typeof"),
-      leftNode
-    ),
+  result.add nkCall.newTree(
+    typeofNode,
     rightNode
   )
-
-proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkPar)
-  for i in 0 ..< node.len():
-    result.add(exprParser.processTSNode(node[i]))
-
-proc processLogicalExpression*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkPar)
-  let child = node[0]
-  var nimSym = ""
-
-  var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip()
-  techo "LOG SYM: ", binarySym
-
-  case binarySym
-  of "!":
-    nimSym = "not"
-  else:
-    raise newException(ExprParseError, &"Unsupported logical symbol \"{binarySym}\"")
-
-  techo "LOG CHILD: ", child.val, ", nim: ", nimSym
-  result.add nkPrefix.newTree(
-    exprParser.state.getIdent(nimSym),
-    exprParser.processTSNode(child)
-  )
-
-proc processBitwiseExpression*(exprParser: ExprParser, node: TSNode): PNode =
-  if node.len() > 1:
-    result = newNode(nkInfix)
-    let left = node[0]
-    let right = node[1]
-    var nimSym = ""
-
-    var binarySym = exprParser.code[left.tsNodeEndByte() ..< right.tsNodeStartByte()].strip()
-    techo "BIN SYM: ", binarySym
-
-    case binarySym
-    of "|", "||":
-      nimSym = "or"
-    of "&", "&&":
-      nimSym = "and"
-    of "^":
-      nimSym = "xor"
-    else:
-      raise newException(ExprParseError, &"Unsupported binary symbol \"{binarySym}\"")
-
-    result.add exprParser.state.getIdent(nimSym)
-    let
-      leftNode = exprParser.processTSNode(left)
-      rightNode = exprParser.processTSNode(right)
-
-    result.add leftNode
-    result.add nkCast.newTree(
-      nkCall.newTree(
-        exprParser.state.getIdent("typeof"),
-        leftNode
-      ),
-      rightNode
+  if binarySym == "/":
+    # Special case. Nim's operators generally output
+    # the same type they take in, except for division.
+    # So we need to emulate C here and cast the whole
+    # expression to the type of the first arg
+    result = nkCall.newTree(
+      typeofNode,
+      result
     )
 
-  elif node.len() == 1:
-    result = newNode(nkPar)
-    let child = node[0]
-    var nimSym = ""
+proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  # Input => !a
+  #
+  # (logical_expression 1 1 2 "!a"
+  #  (identifier 1 2 1 "a")
+  # )
+  #
+  # Output => (not a)
+  #
+  # nkPar(
+  #  nkPrefix(
+  #   nkIdent("not"),
+  #   nkIdent("a")
+  #  )
+  # )
+  result = newNode(nkPar)
 
-    var binarySym = exprParser.code[node.tsNodeStartByte() ..< child.tsNodeStartByte()].strip()
-    techo "BIN SYM: ", binarySym
+  let
+    child = node[0]
+    unarySym = node.tsNodeChild(0).val.strip()
+    nimSym = getNimUnarySym(unarySym)
 
-    case binarySym
-    of "~":
-      nimSym = "not"
-    else:
-      raise newException(ExprParseError, &"Unsupported unary symbol \"{binarySym}\"")
+  if nimSym == "-":
+    # Special case. The minus symbol must be in front of an integer,
+    # so we have to make a gentle cast here to coerce it to one.
+    # Might be bad because we are overwriting the type
+    # There's probably a better way of doing this
+    if typeofNode.isNil:
+      typeofNode = gState.getIdent("int64")
 
     result.add nkPrefix.newTree(
-      exprParser.state.getIdent(nimSym),
-      exprParser.processTSNode(child)
+      gState.getIdent(unarySym),
+      nkPar.newTree(
+        nkCall.newTree(
+          gState.getIdent("int64"),
+          gState.processTSNode(child, typeofNode)
+        )
+      )
     )
   else:
-    raise newException(ExprParseError, &"Invalid bitwise_expression \"{node.val}\"")
+    result.add nkPrefix.newTree(
+      gState.getIdent(nimSym),
+      gState.processTSNode(child, typeofNode)
+    )
 
-proc processTSNode*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkNilLit)
+proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  ## Processes both unary (-1, ~true, !something) and binary (a + b, c * d) expressions
+  if node.len > 1:
+    # Node has left and right children ie: (2 + 7)
+    result = processBinaryExpression(gState, node, typeofNode)
+  elif node.len() == 1:
+    # Node has only one child, ie -(20 + 7)
+    result = processUnaryExpression(gState, node, typeofNode)
+  else:
+    raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"")
+
+proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  # Input => sizeof(int)
+  #
+  # (sizeof_expression 1 1 11 "sizeof(int)"
+  #  (type_descriptor 1 8 3 "int"
+  #   (primitive_type 1 8 3 "int")
+  #  )
+  # )
+  #
+  # Output => sizeof(cint)
+  #
+  # nkCall(
+  #  nkIdent("sizeof"),
+  #  nkIdent("cint")
+  # )
+  result = nkCall.newTree(
+    gState.getIdent("sizeof"),
+    gState.processTSNode(node[0], typeofNode)
+  )
+
+proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode =
+  ## Handle all of the types of expressions here. This proc gets called recursively
+  ## in the processX procs and will drill down to sub nodes.
+  result = newNode(nkNone)
   let nodeName = node.getName()
-  techo "NODE: ", nodeName, ", VAL: ", node.val
+
+  decho "NODE: ", nodeName, ", VAL: ", node.val
+
   case nodeName
   of "number_literal":
-    result = exprParser.processNumberLiteral(node)
+    # Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f
+    # Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334
+    result = gState.processNumberLiteral(node)
   of "string_literal":
-    result = exprParser.processStringLiteral(node)
+    # Input -> "foo\0\x42"
+    # Output -> "foo\0"
+    result = gState.processStringLiteral(node)
   of "char_literal":
-    result = exprParser.processCharacterLiteral(node)
+    # Input -> 'F', '\060' // Octal, '\x5A' // Hex, '\r' // escape sequences
+    # Output -> 'F', '0', 'Z', '\r'
+    result = gState.processCharacterLiteral(node)
   of "expression_statement", "ERROR", "translation_unit":
-    # This may be wrong. What can be in an expression?
-    if node.len > 0:
-      result = exprParser.processTSNode(node[0])
+    # Note that we're parsing partial expressions, so the TSNode might contain
+    # an ERROR node. If that's the case, they usually contain children with
+    # partial results, which will contain parsed expressions
+    #
+    # Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0)
+    # Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ
+    if node.len == 1:
+      result = gState.processTSNode(node[0], typeofNode)
+    elif node.len > 1:
+      var nodes: seq[PNode]
+      for i in 0 ..< node.len:
+        let subNode = gState.processTSNode(node[i], typeofNode)
+        if subNode.kind != nkNone:
+          nodes.add(subNode)
+          # Multiple nodes can get tricky. Don't support them yet, unless they
+          # have at most one valid node
+          if nodes.len > 1:
+            raise newException(ExprParseError, &"Node type \"{nodeName}\" with val ({node.val}) has more than one non empty node")
+      if nodes.len == 1:
+        result = nodes[0]
     else:
       raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children")
-
   of "parenthesized_expression":
-    result = exprParser.processParenthesizedExpr(node)
-  of "bitwise_expression":
-    result = exprParser.processBitwiseExpression(node)
-  of "shift_expression":
-    result = exprParser.processShiftExpression(node)
-  of "logical_expression":
-    result = exprParser.processLogicalExpression(node)
+    # Input -> (IDENT - OTHERIDENT)
+    # Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int)
+    result = gState.processParenthesizedExpr(node, typeofNode)
+  of "sizeof_expression":
+    # Input -> sizeof(char)
+    # Output -> sizeof(cchar)
+    result = gState.processSizeofExpression(node, typeofNode)
+  # binary_expression from the new treesitter upgrade should work here
+  # once we upgrade
+  of "math_expression", "logical_expression", "relational_expression",
+     "bitwise_expression", "equality_expression", "binary_expression",
+     "shift_expression":
+    # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b
+    # Output ->
+    #   typeof(a)(a == typeof(a)(b))
+    #   typeof(a)(a != typeof(a)(b))
+    #   (not a)
+    #   (not a)
+    #   typeof(a)(a < typeof(a)(b))
+    #   typeof(a)(a > typeof(a)(b))
+    #   typeof(a)(a <= typeof(a)(b))
+    #   typeof(a)(a >= typeof(a)(b))
+    #   a shr typeof(a)(b)
+    #   a shl typeof(a)(b)
+    result = gState.processUnaryOrBinaryExpression(node, typeofNode)
+  of "cast_expression":
+    # Input -> (int) a
+    # Output -> cast[cint](a)
+    result = gState.processCastExpression(node, typeofNode)
+  # Why are these node types named true/false?
+  of "true", "false":
+    # Input -> true, false
+    # Output -> true, false
+    result = gState.parseString(node.val)
+  of "type_descriptor", "sized_type_specifier":
+    # Input -> int, unsigned int, long int, etc
+    # Output -> cint, cuint, clong, etc
+    let ty = getType(node.val)
+    if ty.len > 0:
+      # If ty is not empty, one of C's builtin types has been found
+      result = gState.getExprIdent(ty, nskType, parent=node.getName())
+    else:
+      result = gState.getExprIdent(node.val, nskType, parent=node.getName())
+    if result.kind == nkNone:
+      raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"")
   of "identifier":
-    var ident = node.val
-    if ident != "_":
-      ident = exprParser.state.getIdentifier(ident, nskConst)
-    result = exprParser.state.getIdent(ident)
+    # Input -> IDENT
+    # Output -> IDENT (if found in sym table, else error)
+    result = gState.getExprIdent(node, parent=node.getName())
+    if result.kind == nkNone:
+      raise newException(ExprParseError, &"Missing identifier \"{node.val}\"")
+  of "comment":
+    discard
   else:
     raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"")
 
-  techo "NODERES: ", result
+  decho "NODE RESULT: ", result
 
-proc codeToNode*(state: NimState, code: string): PNode =
-  let exprParser = newExprParser(state, code)
+proc parseCExpression*(gState: State, codeRoot: TSNode, name = ""): PNode =
+  ## Parse a c expression from a root ts node
+
+  # This var is used for keeping track of the type of the first
+  # symbol used for type casting
+  var tnode: PNode = nil
+  result = newNode(nkNone)
   try:
-    withCodeAst(exprParser):
-      result = exprParser.processTSNode(root)
+    result = gState.processTSNode(codeRoot, tnode)
   except ExprParseError as e:
-    techo e.msg
-    result = newNode(nkNilLit)
+    decho e.msg
+    result = newNode(nkNone)
   except Exception as e:
-    techo e.msg
-    result = newNode(nkNilLit)
+    decho "UNEXPECTED EXCEPTION: ", e.msg
+    result = newNode(nkNone)
+
+proc parseCExpression*(gState: State, code: string, name = ""): PNode =
+  ## Convert the C string to a nim PNode tree
+  gState.currentExpr = code
+  gState.currentTyCastName = name
+
+  withCodeAst(gState.currentExpr, gState.mode):
+    result = gState.parseCExpression(root, name)
+
+  # Clear the state
+  gState.currentExpr = ""
+  gState.currentTyCastName = ""

nimterop/getters.nim

@@ -1,4 +1,5 @@
 import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times
+import algorithm
 
 import regex
 
@@ -399,9 +400,9 @@ proc printLisp*(code: var string, root: TSNode): string =
     else:
       break
 
-    if node.tsNodeNamedChildCount() != 0:
+    if node.len() != 0:
       result &= "\n"
-      nextnode = node.tsNodeNamedChild(0)
+      nextnode = node[0]
       depth += 1
     else:
       result &= ")\n"
@@ -431,18 +432,21 @@ proc printLisp*(gState: State, root: TSNode): string =
 proc getCommented*(str: string): string =
   "\n# " & str.strip().replace("\n", "\n# ")
 
+proc getDocStrCommented*(str: string): string =
+  "\n## " & str.strip().replace("\n", "\n## ")
+
 proc printTree*(gState: State, pnode: PNode, offset = ""): string =
-  if gState.debug and pnode.kind != nkNone:
+  if not pnode.isNil and gState.debug and pnode.kind != nkNone:
     result &= "\n# " & offset & $pnode.kind & "("
     case pnode.kind
     of nkCharLit:
-      result &= "'" & pnode.intVal.char & "')"
+      result &= ($pnode.intVal.char).escape & ")"
     of nkIntLit..nkUInt64Lit:
       result &= $pnode.intVal & ")"
     of nkFloatLit..nkFloat128Lit:
       result &= $pnode.floatVal & ")"
     of nkStrLit..nkTripleStrLit:
-      result &= "\"" & pnode.strVal & "\")"
+      result &= pnode.strVal.escape & ")"
     of nkSym:
       result &= $pnode.sym & ")"
     of nkIdent:
@@ -461,13 +465,13 @@ proc printTree*(gState: State, pnode: PNode, offset = ""): string =
 
 proc printDebug*(gState: State, node: TSNode) =
   if gState.debug:
-    gecho ("Input => " & gState.getNodeVal(node)).getCommented() & "\n" &
-          gState.printLisp(node).getCommented()
+    gecho ("Input => " & gState.getNodeVal(node)).getCommented()
+    gecho gState.printLisp(node).getCommented()
 
 proc printDebug*(gState: State, pnode: PNode) =
-  if gState.debug:
-    gecho ("Output => " & $pnode).getCommented() & "\n" &
-          gState.printTree(pnode)
+  if gState.debug and pnode.kind != nkNone:
+    gecho ("Output => " & $pnode).getCommented()
+    gecho gState.printTree(pnode)
 
 # Compiler shortcuts
 
@@ -609,6 +613,40 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool
   if result.kind != exactlyOne:
     result.name = result.name[0 .. ^2]
 
+proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string =
+  if commentNodes.len > 0:
+    result = "::"
+    for commentNode in commentNodes:
+      result &= "\n  " & gState.getNodeVal(commentNode).replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "").replace("\n", "\n  ").strip()
+
+proc getPrevCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] =
+  ## Here we want to go until the node we get is not a comment
+  ## for cases with multiple ``//`` comments instead of one ``/* */``
+  ## section
+  var sibling = node.tsNodePrevNamedSibling()
+  var i = 0
+  while not sibling.isNil and i < maxSearch:
+    while not sibling.isNil and sibling.getName() == "comment":
+      result.add(sibling)
+      sibling = sibling.tsNodePrevNamedSibling()
+    if sibling.isNil:
+      result.reverse
+      return
+    sibling = sibling.tsNodePrevNamedSibling()
+    i += 1
+
+  result.reverse
+
+proc getNextCommentNodes*(node: TSNode, maxSearch=1): seq[TSNode] =
+  ## We only want to search for the next comment node (ie: inline)
+  var sibling = node.tsNodeNextNamedSibling()
+  var i = 0
+  while not sibling.isNil and i < maxSearch:
+    if sibling.getName() == "comment":
+      return @[sibling]
+    sibling = sibling.tsNodeNextNamedSibling()
+    i += 1
+
 proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
   if node.tsNodeNamedChildCount() != 0:
     for i in 0 .. node.tsNodeNamedChildCount()-1:

nimterop/globals.nim

@@ -1,4 +1,4 @@
-import sequtils, sets, tables
+import sequtils, sets, tables, strutils
 
 import regex
 
@@ -76,6 +76,11 @@ type
     # All const names for enum casting
     constIdentifiers*: HashSet[string]
 
+    # All symbols that have been skipped due to
+    # being unwrappable or the user provided
+    # override is blank
+    skippedSyms*: HashSet[string]
+
     # Legacy ast fields, remove when ast2 becomes default
     constStr*, enumStr*, procStr*, typeStr*: string
 
@@ -93,6 +98,9 @@ type
 
     currentHeader*, impShort*, sourceFile*: string
 
+    # Used for the exprparser.nim module
+    currentExpr*, currentTyCastName*: string
+
     data*: seq[tuple[name, val: string]]
 
     nodeBranch*: seq[string]
@@ -113,12 +121,12 @@ when not declared(CIMPORT):
   export gAtoms, gExpressions, gEnumVals, Kind, Ast, AstTable, State, nBl, Bl
 
   # Redirect output to file when required
-  template gecho*(args: string) {.dirty.} =
+  template gecho*(args: string) =
     if gState.outputHandle.isNil:
       echo args
     else:
       gState.outputHandle.writeLine(args)
 
-  template decho*(str: untyped): untyped =
+  template decho*(args: varargs[string, `$`]): untyped =
     if gState.debug:
-      gecho str.getCommented()
+      gecho join(args, "").getCommented()

nimterop/toast.nim

@@ -2,16 +2,11 @@ import os, osproc, strformat, strutils, tables, times
 
 import "."/treesitter/[api, c, cpp]
 
-import "."/[ast, ast2, globals, getters, grammar, build]
+import "."/[ast, ast2, globals, getters, grammar, build, tshelp]
 
 proc process(gState: State, path: string, astTable: AstTable) =
   doAssert existsFile(path), &"Invalid path {path}"
 
-  var parser = tsParserNew()
-
-  defer:
-    parser.tsParserDelete()
-
   if gState.mode.Bl:
     gState.mode = getCompilerMode(path)
 
@@ -20,31 +15,16 @@ proc process(gState: State, path: string, astTable: AstTable) =
   else:
     gState.code = readFile(path)
 
-  doAssert gState.code.nBl, "Empty file or preprocessor error"
-
-  if gState.mode == "c":
-    doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser"
-  elif gState.mode == "cpp":
-    doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
-  else:
-    doAssert false, &"Invalid parser {gState.mode}"
-
-  var
-    tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32)
-    root = tree.tsTreeRootNode()
-
-  defer:
-    tree.tsTreeDelete()
-
-  if gState.past:
-    gecho gState.printLisp(root)
-  elif gState.pnim:
-    if Feature.ast2 in gState.feature:
-      ast2.parseNim(gState, path, root)
-    else:
-      ast.parseNim(gState, path, root, astTable)
-  elif gState.preprocess:
-    gecho gState.code
+  withCodeAst(gState.code, gState.mode):
+    if gState.past:
+      gecho gState.printLisp(root)
+    elif gState.pnim:
+      if Feature.ast2 in gState.feature:
+        ast2.parseNim(gState, path, root)
+      else:
+        ast.parseNim(gState, path, root, astTable)
+    elif gState.preprocess:
+      gecho gState.code
 
 # CLI processing with default values
 proc main(

nimterop/tshelp.nim

@@ -0,0 +1,28 @@
+import "."/treesitter/[c, cpp]
+
+template withCodeAst*(code: string, mode: string, body: untyped): untyped =
+  ## A simple template to inject the TSNode into a body of code
+  mixin treeSitterC
+  mixin treeSitterCpp
+
+  var parser = tsParserNew()
+  defer:
+    parser.tsParserDelete()
+
+  doAssert code.nBl, "Empty code or preprocessor error"
+
+  if mode == "c":
+    doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser"
+  elif mode == "cpp":
+    doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
+  else:
+    doAssert false, "Invalid parser " & mode
+
+  var
+    tree = parser.tsParserParseString(nil, code.cstring, code.len.uint32)
+    root {.inject.} = tree.tsTreeRootNode()
+
+  body
+
+  defer:
+    tree.tsTreeDelete()

tests/include/tast2.h

@@ -8,6 +8,42 @@ extern "C" {
 #define D "hello"
 #define E 'c'
 
+#define UEXPR (1234u << 1)
+#define ULEXPR (1234ul << 2)
+#define ULLEXPR (1234ull << 3)
+#define LEXPR (1234l << 4)
+#define LLEXPR (1234ll << 5)
+
+#define SHL1 (1u << 1)
+#define SHL2 (1u << 2)
+#define SHL3 (1u << 3)
+#define COERCE 645635634896ull + 35436
+#define COERCE2 645635634896 + 35436ul
+#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1)
+#define BOOL true
+#define MATHEXPR (1 + 2/3*20 - 100)
+#define ANDEXPR (100 & 11000)
+#define CASTEXPR (char) 34
+#define AVAL 100
+#define BVAL 200
+#define EQ1 AVAL <= BVAL
+#define EQ2 AVAL >= BVAL
+#define EQ3 AVAL > BVAL
+#define EQ4 AVAL < BVAL
+#define EQ5 AVAL != BVAL
+#define EQ6 AVAL == BVAL
+
+#define SIZEOF sizeof(char)
+#define REG_STR "regular string"
+#define NOTSUPPORTEDSTR "not a " REG_STR
+
+#define NULLCHAR '\0'/* comments should not break things*/
+#define OCTCHAR '\012' // nor should this comment
+#define HEXCHAR '\xFE'
+#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar"
+
+#define ALLSHL (SHL1 | SHL2 | SHL3)
+
 struct A0;
 struct A1 {};
 typedef struct A2;

tests/tast2.nim

@@ -3,6 +3,10 @@ import macros, os, sets, strutils
 import nimterop/[cimport]
 
 static:
+  # Skip casting on lower nim compilers because
+  # the VM does not support it
+  when (NimMajor, NimMinor, NimPatch) < (1, 0, 0):
+    cSkipSymbol @["CASTEXPR"]
   cDebug()
 
 const
@@ -93,11 +97,11 @@ macro testFields(t: typed, fields: static[string] = "") =
     for i in 0 ..< rl.len:
       let
         name = ($rl[i][0]).strip(chars = {'*'})
-        typ = ($(rl[i][1].repr())).replace("\n", "").replace("  ", "")
+        typ = ($(rl[i][1].repr())).replace("\n", "").replace("  ", "").replace("typeof", "type")
         n = names.find(name)
       assert n != -1, $t & "." & name & " invalid"
-      assert types[n] == typ,
-        "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ
+      assert types[n].replace("typeof", "type") == typ,
+        "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ
 
 assert A == 2
 assert B == 1.0
@@ -105,6 +109,48 @@ assert C == 0x10
 assert D == "hello"
 assert E == 'c'
 
+assert not defined(NOTSUPPORTEDSTR)
+
+assert UEXPR == (1234.uint shl 1)
+assert ULEXPR == (1234.uint32 shl 2)
+assert ULLEXPR == (1234.uint64 shl 3)
+assert LEXPR == (1234.int32 shl 4)
+assert LLEXPR == (1234.int64 shl 5)
+
+assert AVAL == 100
+assert BVAL == 200
+
+assert EQ1 == (AVAL <= BVAL)
+assert EQ2 == (AVAL >= BVAL)
+assert EQ3 == (AVAL > BVAL)
+assert EQ4 == (AVAL < BVAL)
+assert EQ5 == (AVAL != BVAL)
+assert EQ6 == (AVAL == BVAL)
+
+assert SIZEOF == 1
+
+assert COERCE == 645635670332'u64
+assert COERCE2 == 645635670332'i64
+
+assert BINEXPR == 5
+assert BOOL == true
+assert MATHEXPR == -99
+assert ANDEXPR == 96
+
+when (NimMajor, NimMinor, NimPatch) >= (1, 0, 0):
+  assert CASTEXPR == 34.chr
+
+assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar"
+assert NULLCHAR == '\0'
+assert OCTCHAR == '\n'
+assert HEXCHAR.int == 0xFE
+
+assert SHL1 == (1.uint shl 1)
+assert SHL2 == (1.uint shl 2)
+assert SHL3 == (1.uint shl 3)
+
+assert ALLSHL == (SHL1 or SHL2 or SHL3)
+
 assert A0 is object
 testFields(A0, "f1!cint")
 checkPragmas(A0, pHeaderBy, istype = false)
@@ -271,7 +317,7 @@ var a21p: A21p
 a21p = addr a20
 
 assert A22 is object
-testFields(A22, "f1|f2!ptr ptr cint|array[123 + 132, ptr cint]")
+testFields(A22, "f1|f2!ptr ptr cint|array[123 + type(123)(132), ptr cint]")
 checkPragmas(A22, pHeaderBy, istype = false)
 var a22: A22
 a22.f1 = addr a15.a2[0]
@@ -427,4 +473,4 @@ checkPragmas(nested, pHeaderImpBy)
 
 when defined(HEADER):
   assert sitest1(5) == 10
-  assert sitest1(10) == 20
+  assert sitest1(10) == 20

tests/tmath.nim

@@ -13,6 +13,12 @@ when defined(windows):
       complex = object
 
 static:
+  when (NimMajor, NimMinor, NimPatch) < (1, 0, 0):
+    # FP_ILOGB0 and FP_ILOGBNAN are casts that are unsupported
+    # on lower Nim VMs
+    cSkipSymbol @["math_errhandling", "FP_ILOGB0", "FP_ILOGBNAN"]
+  else:
+    cSkipSymbol @["math_errhandling"]
   cDebug()
   cDisableCaching()
   cAddStdDir()