ast2 test fix for osx, catch parseString errors, handle type field comments

nimterop.nimble

@@ -19,14 +19,17 @@ proc execCmd(cmd: string) =
   exec cmd
 
 proc execTest(test: string, flags = "") =
-  execCmd "nim c -f " & flags & " -r " & test
-  execCmd "nim cpp " & flags & " -r " & test
+  execCmd "nim c --hints:off -f " & flags & " -r " & test
+  execCmd "nim cpp --hints:off " & flags & " -r " & test
 
 task buildToast, "build toast":
-  execCmd("nim c -f nimterop/toast.nim")
+  execCmd("nim c --hints:off -f nimterop/toast.nim")
 
 task bt, "build toast":
-  execCmd("nim c -d:danger nimterop/toast.nim")
+  execCmd("nim c --hints:off -d:danger nimterop/toast.nim")
+
+task btd, "build toast":
+  execCmd("nim c --hints:off nimterop/toast.nim")
 
 task docs, "Generate docs":
   buildDocs(@["nimterop/all.nim"], "build/htmldocs")
@@ -38,13 +41,16 @@ task test, "Test":
   execTest "tests/tast2.nim", "-d:HEADER"
 
   execTest "tests/tnimterop_c.nim"
-  execTest "tests/tnimterop_c.nim", "-d:AST2"
-  execTest "tests/tnimterop_c.nim", "-d:HEADER -d:AST2"
+  execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\""
+  execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\""
 
-  execCmd "nim cpp -f -r tests/tnimterop_cpp.nim"
+  execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim"
   execCmd "./nimterop/toast -pnk -E=_ tests/include/toast.h"
   execCmd "./nimterop/toast -pnk -E=_ -f:ast2 tests/include/toast.h"
+
   execTest "tests/tpcre.nim"
+  #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\""
+  #execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2 -H\""
 
   # Platform specific tests
   when defined(Windows):

nimterop/ast2.nim

@@ -2,7 +2,7 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times
 
 import regex
 
-import compiler/[ast, idents, modulegraphs, options, parser, renderer]
+import compiler/[ast, idents, lineinfos, modulegraphs, options, parser, renderer]
 
 import "."/treesitter/api
 
@@ -17,9 +17,19 @@ proc getPtrType*(str: string): string =
     else:
       str
 
+proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) =
+  # Raise exception in parseString() instead of exiting
+  raise newException(Exception, "")
+
 proc parseString(nimState: NimState, str: string): PNode =
-  # Parse a string into Nim AST
-  result = parseString(str, nimState.identCache, nimState.config)
+  # Parse a string into Nim AST - use custom error handler that raises
+  # an exception rather than exiting on failure
+  try:
+    result = parseString(
+      str, nimState.identCache, nimState.config, errorHandler = handleError
+    )
+  except:
+    discard
 
 proc getLit*(nimState: NimState, str: string): PNode =
   # Used to convert #define literals into const and expressions
@@ -33,9 +43,8 @@ proc getLit*(nimState: NimState, str: string): PNode =
   elif str.contains(re"^[\-]?[\d]*[.]?[\d]+$"):   # float
     result = newFloatNode(nkFloatLit, parseFloat(str))
 
-  # # TODO - hex becomes int on render
-  # elif str.contains(re"^0x[\da-fA-F]+$"):         # hexadecimal
-  #   result = newIntNode(nkIntLit, parseHexInt(str))
+  elif str.contains(re"^0x[\da-fA-F]+$"):         # hexadecimal
+    result = nimState.parseString(str)
 
   elif str.contains(re"^'[[:ascii:]]'$"):         # char
     result = newNode(nkCharLit)
@@ -46,8 +55,9 @@ proc getLit*(nimState: NimState, str: string): PNode =
 
   else:
     result = nimState.parseString(nimState.getNimExpression(str))
-    if result.isNil:
-      result = newNode(nkNilLit)
+
+  if result.isNil:
+    result = newNode(nkNilLit)
 
 proc getOverrideOrSkip(nimState: NimState, node: TSNode, origname: string, kind: NimSymKind): PNode =
   # Check if symbol `origname` of `kind` and `origname` has any cOverride defined
@@ -132,7 +142,9 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode
   if name.Bl:
     # Name skipped or overridden since blank
     result = nimState.getOverrideOrSkip(node, origname, nskConst)
-  elif valident.kind != nkNilLit:
+  elif valident.kind in {nkCharLit .. nkStrLit} or
+    (valident.kind == nkStmtList and valident.len > 0 and
+    valident[0].kind in {nkCharLit .. nkStrLit}):
     if nimState.addNewIdentifer(name):
       # const X* = Y
       #
@@ -147,7 +159,11 @@ proc newConstDef(nimState: NimState, node: TSNode, fname = "", fval = ""): PNode
       result = newNode(nkConstDef)
       result.add ident
       result.add newNode(nkEmpty)
-      result.add valident
+      if valident.kind == nkStmtList and valident.len == 1:
+        # Collapse single line statement
+        result.add valident[0]
+      else:
+        result.add valident
     else:
       necho &"# const '{origname}' is duplicate, skipped"
   else:
@@ -576,11 +592,12 @@ proc newRecListTree(nimState: NimState, name: string, node: TSNode): PNode =
     result = newNode(nkRecList)
 
     for i in 0 ..< node.len:
-      # Add nkIdentDefs for each field
-      let
-        field = nimState.newIdentDefs(name, node[i], i, exported = true)
-      if not field.isNil:
-        result.add field
+      if node[i].getName() == "field_declaration":
+        # Add nkIdentDefs for each field
+        let
+          field = nimState.newIdentDefs(name, node[i], i, exported = true)
+        if not field.isNil:
+          result.add field
 
 proc addTypeObject(nimState: NimState, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) =
   # Add a type of object

tests/include/tast2.h

@@ -45,11 +45,11 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p;
 typedef struct A20 { char a1; } A20, A21, *A21p;
 
 //Expression
-typedef struct A22 { int **f1; int *f2[123+132]; } A22;
+typedef struct A22 { const int **f1; int *f2[123+132]; } A22;
 
 //Unions
 union U1 {int f1; float f2; };
-typedef union U2 { int **f1; int abc[123+132]; } U2;
+typedef union U2 { const int **f1; int abc[123+132]; } U2;
 
 // Enums
 
@@ -157,11 +157,11 @@ typedef struct { char *a1; int *a2[1]; } A19, *A19p;
 typedef struct A20 { char a1; } A20, A21, *A21p;
 
 //Expression
-typedef struct A22 { int **f1; int *f2[123+132]; } A22;
+typedef struct A22 { const int **f1; int *f2[123+132]; } A22;
 
 //Unions
 union U1 {int f1; float f2; };
-typedef union U2 { int **f1; int abc[123+132]; } U2;
+typedef union U2 { const int **f1; int abc[123+132]; } U2;
 
 // Enums
 
@@ -220,4 +220,4 @@ typedef enum VSPresetFormat {
 
 #ifdef __cplusplus
 }
-#endif
+#endif

tests/tnimterop_c.nim

@@ -41,17 +41,8 @@ cOverride:
   proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.}
   proc weirdfunc2(mango: ptr ptr cchar): int {.importc.}
 
-# includeHeader
-const header =
-  when defined(HEADER): " -H"
-  else: ""
-
-# Test AST2
-const mode =
-  when defined(AST2): " -f:ast2"
-  else: ""
-
-cImport(cSearchPath("test.h"), flags = header & mode)
+const FLAGS {.strdefine.} = ""
+cImport(cSearchPath("test.h"), flags = FLAGS)
 
 check TEST_INT == 512
 check TEST_FLOAT == 5.12

tests/tpcre.nim

@@ -30,7 +30,8 @@ cPlugin:
   proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
     sym.name = sym.name.replace("pcre_", "")
 
-cImport(pcreH, dynlib="dynpcre")
+const FLAGS {.strdefine.} = ""
+cImport(pcreH, dynlib="dynpcre", flags = FLAGS)
 
 echo version()