Add string and char support

Update some comments

Rename exprparser main proc

Don't export parse procs

Add missing utils module

Try to fix array type test

Try fix cast test error

Disable cast test for now

Revert back comment test. Have to figure out how to test without vm

nimterop/ast2.nim

@@ -18,7 +18,7 @@ proc getPtrType*(str: string): string =
       str
 
 proc getLit*(nimState: NimState, str: string, expression = false): PNode =
-  result = nimState.codeToNode(str)
+  result = nimState.parseCExpression(str)
 
 proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode =
   # Check if symbol `origname` of `kind` and `origname` has any cOverride defined
@@ -164,7 +164,7 @@ proc addPragma(gState: State, node: TSNode, pragma: PNode, name: string, value:
   if value.isNil:
     pragma.add pident
   else:
-    var
+    let
       colExpr = newNode(nkExprColonExpr)
     colExpr.add pident
     colExpr.add value
@@ -1386,7 +1386,7 @@ proc addEnum(gState: State, node: TSNode) =
 
           if en.len > 1 and en[1].getName() in gEnumVals:
             # Explicit value
-            fval = "(" & gState.getNimExpression(gState.getNodeVal(en[1]), name) & ")." & name
+            fval = "(" & $gState.parseCExpression(gState.getNodeVal(en[1]), name) & ")." & name
 
           # Cannot use newConstDef() since parseString(fval) adds backticks to and/or
           gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0]

nimterop/exprparser.nim

@@ -1,4 +1,4 @@
-import strformat, strutils, macros
+import strformat, strutils, macros, sets
 
 import regex
 
@@ -12,11 +12,12 @@ type
   ExprParser* = ref object
     state*: NimState
     code*: string
+    name*: string
 
   ExprParseError* = object of CatchableError
 
-proc newExprParser*(state: NimState, code: string): ExprParser =
-  ExprParser(state: state, code: code)
+proc newExprParser*(state: NimState, code: string, name = ""): ExprParser =
+  ExprParser(state: state, code: code, name: name)
 
 template techo(msg: varargs[string, `$`]) =
   if exprParser.state.gState.debug:
@@ -38,6 +39,8 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent
   if ident != "_":
     # Process the identifier through cPlugin
     ident = exprParser.state.getIdentifier(ident, kind, parent)
+  if exprParser.name.nBl and ident in exprParser.state.constIdentifiers:
+    ident = ident & "." & exprParser.name
   if ident != "":
     result = exprParser.state.getIdent(ident)
 
@@ -70,6 +73,58 @@ template withCodeAst(exprParser: ExprParser, body: untyped): untyped =
   defer:
     tree.tsTreeDelete()
 
+proc parseChar(charStr: string): uint8 {.inline.} =
+  ## Parses a character literal out of a string. This is needed
+  ## because treesitter gives unescaped characters when parsing
+  ## strings.
+  if charStr.len == 1:
+    return charStr[0].uint8
+
+  # Handle octal, hex, unicode?
+  if charStr.startsWith("\\x"):
+    result = parseHexInt(charStr.replace("\\x", "0x")).uint8
+  elif charStr.len == 4: # Octal
+    result = parseOctInt("0o" & charStr[1 ..< charStr.len]).uint8
+
+  if result == 0:
+    case charStr
+    of "\\0":
+      result = ord('\0')
+    of "\\a":
+      result = 0x07
+    of "\\b":
+      result = 0x08
+    of "\\e":
+      result = 0x1B
+    of "\\f":
+      result = 0x0C
+    of "\\n":
+      result = '\n'.uint8
+    of "\\r":
+      result = 0x0D
+    of "\\t":
+      result = 0x09
+    of "\\v":
+      result = 0x0B
+    of "\\\\":
+      result = 0x5C
+    of "\\'":
+      result = '\''.uint8
+    of "\\\"":
+      result = '\"'.uint8
+    of "\\?":
+      result = 0x3F
+    else:
+      discard
+
+  if result > uint8.high:
+    result = uint8.high
+
+proc getCharLit(charStr: string): PNode {.inline.} =
+  ## Convert a character string into a proper Nim char lit node
+  result = newNode(nkCharLit)
+  result.intVal = parseChar(charStr).int64
+
 proc getNumNode(number, suffix: string): PNode {.inline.} =
   ## Convert a C number to a Nim number PNode
   result = newNode(nkNone)
@@ -117,7 +172,7 @@ proc getNumNode(number, suffix: string): PNode {.inline.} =
   else:
     result.intVal = parseInt(number)
 
-proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode =
+proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode =
   ## Parse a number literal from a TSNode. Can be a float, hex, long, etc
   result = newNode(nkNone)
   let nodeVal = node.val
@@ -141,17 +196,29 @@ proc processNumberLiteral*(exprParser: ExprParser, node: TSNode): PNode =
   else:
     raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"")
 
-proc processCharacterLiteral*(exprParser: ExprParser, node: TSNode): PNode =
-  result = newNode(nkCharLit)
-  result.intVal = node.val[1].int64
+proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode =
+  let val = node.val
+  result = getCharLit(val[1 ..< val.len - 1])
 
-proc processStringLiteral*(exprParser: ExprParser, node: TSNode): PNode =
-  let nodeVal = node.val
-  result = newStrNode(nkStrLit, nodeVal[1 ..< nodeVal.len - 1])
+proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode =
+  let
+    nodeVal = node.val
+    strVal = nodeVal[1 ..< nodeVal.len - 1]
 
-proc processTSNode*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode
+  const
+    str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])"
+    reg = re(str)
 
-proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
+  # Convert the c string escape sequences/etc to Nim chars
+  var nimStr = newStringOfCap(nodeVal.len)
+  for m in strVal.findAll(reg):
+    nimStr.add(parseChar(strVal[m.group(0)[0]]).chr)
+
+  result = newStrNode(nkStrLit, nimStr)
+
+proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode
+
+proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
   result = newNode(nkInfix)
   let
     left = node[0]
@@ -185,18 +252,18 @@ proc processShiftExpression*(exprParser: ExprParser, node: TSNode, typeofNode: v
     rightNode
   )
 
-proc processParenthesizedExpr*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
+proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
   result = newNode(nkPar)
   for i in 0 ..< node.len():
     result.add(exprParser.processTSNode(node[i], typeofNode))
 
-proc processCastExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
+proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
   result = nkCast.newTree(
     exprParser.processTSNode(node[0], typeofNode),
     exprParser.processTSNode(node[1], typeofNode)
   )
 
-proc processLogicalExpression*(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
+proc processLogicalExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
   result = newNode(nkPar)
   let child = node[0]
   var nimSym = ""
@@ -415,13 +482,13 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode):
 
   techo "NODE RESULT: ", result
 
-proc codeToNode*(state: NimState, code: string): PNode =
+proc parseCExpression*(state: NimState, code: string, name = ""): PNode =
   ## Convert the C string to a nim PNode tree
   result = newNode(nkNone)
   # This is used for keeping track of the type of the first
   # symbol
   var tnode: PNode = nil
-  let exprParser = newExprParser(state, code)
+  let exprParser = newExprParser(state, code, name)
   try:
     withCodeAst(exprParser):
       result = exprParser.processTSNode(root, tnode)

nimterop/utils.nim

@@ -0,0 +1,18 @@
+import compiler/[ast, lineinfos, msgs, options, parser, renderer]
+
+import "."/[globals, getters]
+
+proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) =
+  # Raise exception in parseString() instead of exiting for errors
+  if msg < warnMin:
+    raise newException(Exception, msgKindToString(msg))
+
+proc parseString*(nimState: NimState, str: string): PNode =
+  # Parse a string into Nim AST - use custom error handler that raises
+  # an exception rather than exiting on failure
+  try:
+    result = parseString(
+      str, nimState.identCache, nimState.config, errorHandler = handleError
+    )
+  except:
+    decho getCurrentExceptionMsg()

tests/include/tast2.h

@@ -23,6 +23,12 @@ extern "C" {
 #define BOOL true
 #define MATHEXPR (1 + 2/3*20 - 100)
 #define ANDEXPR (100 & 11000)
+#define CASTEXPR (int) 34
+
+#define NULLCHAR '\0'
+#define OCTCHAR '\012'
+#define HEXCHAR '\xFE'
+#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar"
 
 #define ALLSHL (SHL1 | SHL2 | SHL3)
 

tests/tast2.nim

@@ -93,11 +93,11 @@ macro testFields(t: typed, fields: static[string] = "") =
     for i in 0 ..< rl.len:
       let
         name = ($rl[i][0]).strip(chars = {'*'})
-        typ = ($(rl[i][1].repr())).replace("\n", "").replace("  ", "")
+        typ = ($(rl[i][1].repr())).replace("\n", "").replace("  ", "").replace("typeof", "type")
         n = names.find(name)
       assert n != -1, $t & "." & name & " invalid"
-      assert types[n] == typ,
-        "typeof(" & $t & ":" & name & ") != " & types[n] & ", is " & typ
+      assert types[n].replace("typeof", "type") == typ,
+        "typeof(" & $t & ":" & name & ") != " & types[n].replace("typeof", "type") & ", is " & typ
 
 assert A == 2
 assert B == 1.0
@@ -118,6 +118,12 @@ assert BINEXPR == 5
 assert BOOL == true
 assert MATHEXPR == -99
 assert ANDEXPR == 96
+assert CASTEXPR == 34.chr
+
+assert TRICKYSTR == "N\x1C\nfoo\x00\'\"\c\v\a\b\e\f\t\\\\?bar"
+assert NULLCHAR == '\0'
+assert OCTCHAR == '\n'
+assert HEXCHAR.int == 0xFE
 
 assert SHL1 == (1.uint shl 1)
 assert SHL2 == (1.uint shl 2)