ast2 handle tree-sitter errors, multi-reorder

nimterop/ast2.nim

@@ -27,7 +27,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS
     if not pnode.isNil:
       result = pnode[0][0]
   else:
-    gecho &"\n# $1'{origname}' skipped" % skind
+    gecho &"# {skind}'{origname}' skipped"
     gState.skippedSyms.incl origname
 
 proc addOverrideFinal(gState: State, kind: NimSymKind) =
@@ -391,7 +391,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas:
 
     gState.identifierNodes[name] = result
   else:
-    gecho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind)
+    gecho &"# {getKeyword(kind)} '{origname}' is duplicate, skipped"
 
 proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode =
   # Create nkBracketExpr tree depending on input
@@ -682,7 +682,7 @@ proc newProcTy(gState: State, name: string, node: TSNode, rtyp: PNode): PNode =
   if node.getVarargs():
     gState.addPragma(node, result[^1], "varargs")
 
-proc processNode(gState: State, node: TSNode): bool
+proc processNode(gState: State, node: TSNode): Status
 proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
   # Create nkRecList tree for specified object
   if not node.isNil:
@@ -720,9 +720,9 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
                 $gState.enumSection[^1][0][1]
               )
             else:
-              (true, "")
+              (success, "")
 
-        if not processed:
+        if processed != success:
           return nil
 
         # Add nkIdentDefs for each field
@@ -1699,35 +1699,52 @@ proc addDef(gState: State, node: TSNode) =
     if not gState.noHeader:
       gState.addProc(node[start+1], node[start], commentNodes)
     else:
-      gecho &"\n# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" %
+      gecho "# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" %
         gState.getNodeVal(node[start+1].getAtom())
 
-proc processNode(gState: State, node: TSNode): bool =
-  result = true
+proc processNode(gState: State, node: TSNode): Status =
+  const
+    known = ["preproc_def", "type_definition",
+      "struct_specifier", "union_specifier", "enum_specifier",
+      "declaration", "function_definition"].toHashSet()
 
-  case node.getName()
-  of "preproc_def":
-    gState.addConst(node)
-  of "type_definition":
-    if node.len > 0 and node[0].getName() == "enum_specifier":
-      gState.addEnum(node)
-    elif node.len > 0 and node[0].getName() == "union_specifier":
-      gState.addType(node, union = true)
+  result = success
+  let
+    name = node.getName()
+  if name in known:
+    # Recognized top-level nodes
+    let
+      err = node.anyChildInTree("ERROR")
+    if not err.isNil:
+      # Bail on errors
+      gState.printDebug(node)
+      gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped"
+      result = Status.error
     else:
-      gState.addType(node)
-  of "struct_specifier":
-    gState.addType(node)
-  of "union_specifier":
-    gState.addType(node, union = true)
-  of "enum_specifier":
-    gState.addEnum(node)
-  of "declaration":
-    gState.addDecl(node)
-  of "function_definition":
-    gState.addDef(node)
+      # Process nodes
+      case name
+      of "preproc_def":
+        gState.addConst(node)
+      of "type_definition":
+        if node.len > 0 and node[0].getName() == "enum_specifier":
+          gState.addEnum(node)
+        elif node.len > 0 and node[0].getName() == "union_specifier":
+          gState.addType(node, union = true)
+        else:
+          gState.addType(node)
+      of "struct_specifier":
+        gState.addType(node)
+      of "union_specifier":
+        gState.addType(node, union = true)
+      of "enum_specifier":
+        gState.addEnum(node)
+      of "declaration":
+        gState.addDecl(node)
+      of "function_definition":
+        gState.addDef(node)
   else:
-    # Unknown
-    result = false
+    # Unknown, will check child nodes
+    result = unknown
 
 proc searchTree(gState: State, root: TSNode) =
   # Search AST generated by tree-sitter for recognized elements
@@ -1735,7 +1752,7 @@ proc searchTree(gState: State, root: TSNode) =
     node = root
     nextnode: TSNode
     depth = 0
-    processed = false
+    processed = success
 
   while true:
     if not node.isNil and depth > -1:
@@ -1743,7 +1760,7 @@ proc searchTree(gState: State, root: TSNode) =
     else:
       break
 
-    if not processed and node.len != 0:
+    if processed == unknown and node.len != 0:
       nextnode = node[0]
       depth += 1
     else:
@@ -1773,27 +1790,28 @@ proc searchTree(gState: State, root: TSNode) =
 proc setupPragmas(gState: State, root: TSNode, fullpath: string) =
   # Create shortcut pragmas to reduce clutter
   var
-    hdrPragma: PNode
-    dynPragma: PNode
+    count = 0
 
   if not gState.noHeader:
     # {.pragma: impnameHdr, header: "xxx".}
-    hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr"))
+    let
+      hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr"))
     gState.addPragma(root, hdrPragma, "header", newStrNode(nkStrLit, fullpath))
+    gState.pragmaSection.add hdrPragma
+    count += 1
 
   if gState.dynlib.nBl:
     # {.pragma: impnameDyn, dynlib: libname.}
-    dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn"))
+    let
+      dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn"))
     gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib))
-
-  # Add pragma shortcuts to output
-  if not hdrPragma.isNil:
-    gState.pragmaSection.add hdrPragma
-  if not dynPragma.isNil:
     gState.pragmaSection.add dynPragma
+    count += 1
 
   # Add `{.experimental: "codeReordering".} for #206
-  gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering"))
+  if gState.pragmaSection.len == count:
+    # Only if not already done
+    gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering"))
 
 proc initNim*(gState: State) =
   # Initialize for parseNim() one time

nimterop/getters.nim

@@ -251,7 +251,6 @@ proc getPreprocessor*(gState: State, fullpath: string) =
     cmts = if gState.noComments: "" else: "-CC"
     cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """
 
-    ddata: seq[string]
     rdata: seq[string]
     start = false
     sfile = fullpath.sanitizePath(noQuote = true)
@@ -302,11 +301,8 @@ proc getPreprocessor*(gState: State, fullpath: string) =
       if start:
         if "#undef" in line:
           continue
-        elif line.startsWith("#define"):
-          ddata.add line
-        else:
-          rdata.add line
-  gState.code = ddata.join("\n") & "\n" & rdata.join("\n")
+        rdata.add line
+  gState.code = rdata.join("\n")
 
 converter toString*(kind: Kind): string =
   return case kind:

nimterop/globals.nim

@@ -131,6 +131,9 @@ when defined(TOAST):
 
     AstTable* {.used.} = TableRef[string, seq[ref Ast]]
 
+    Status* = enum
+      success, unknown, error
+
   # Redirect output to file when required
   template gecho*(args: string) =
     if gState.outputHandle.isNil: