Reduce regex for perf, reduce globals for CT

nimterop.nimble

@@ -30,15 +30,15 @@ proc execTest(test: string, flags = "", runDocs = true) =
     mkDir docPath
     buildDocs(@[test], docPath, nimArgs = "--hints:off " & flags)
 
-task buildToast, "build toast":
-  execCmd("nim c --hints:off nimterop/toast.nim")
-
 task buildTimeit, "build timer":
   exec "nim c --hints:off -d:danger tests/timeit"
 
-task bt, "build toast":
+task buildToast, "build toast":
   execCmd("nim c --hints:off -d:danger nimterop/toast.nim")
 
+task bt, "build toast":
+  buildToastTask()
+
 task btd, "build toast":
   execCmd("nim c -g nimterop/toast.nim")
 

nimterop/build.nim

@@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables
 
 import os except findExe, sleep
 
-import regex
-
 type
   BuildType* = enum
     btAutoconf, btCmake
@@ -995,7 +993,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
   ## prior to the build process.
   var
     origname = header.extractFilename().split(".")[0]
-    name = origname.replace(re"[[:^alnum:]]", "")
+    name = origname.split(seps = AllChars-Letters-Digits).join()
 
     # -d:xxx for this header
     stdStr = name & "Std"

nimterop/cimport.nim

@@ -17,8 +17,6 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using:
 
 import hashes, macros, os, strformat, strutils
 
-import regex
-
 import "."/[build, globals, paths, types]
 export types
 
@@ -684,7 +682,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib:
     hash = output.hash().abs()
     hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h"
     npath = hpath[0 .. hpath.rfind('.')] & "nim"
-    header = ("header" & fullpath.splitFile().name.replace(re"[-.]+", ""))
+    header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join()
 
   if not fileExists(hpath) or gStateCT.nocache or compileOption("forceBuild"):
     mkDir(hpath.parentDir())
@@ -715,10 +713,6 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib:
   var
     nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath)
 
-  nimout = nimout.
-    replace(re"([u]?int[\d]+)_t", "$1").
-    replace(re"([u]?int)ptr_t", "ptr $1")
-
   if gStateCT.debug:
     echo nimout
 

nimterop/getters.nim

@@ -227,7 +227,7 @@ proc getOverride*(gState: State, name: string, kind: NimSymKind): string =
         result = sym.override
 
         if kind != nskProc:
-          result = result.replace(re"(?m)^(.*?)$", "  $1")
+          result = "  " & result.replace("\n", "\n ")
 
 proc getOverrideFinal*(gState: State, kind: NimSymKind): string =
   # Get all unused cOverride symbols of `kind`

nimterop/globals.nim

@@ -1,58 +1,19 @@
-import sequtils, sets, tables, strutils
-
-import regex
-
-import "."/plugin
+import tables
 
 when defined(TOAST):
+  import sets, sequtils, strutils
+
+  import regex
+
+  import "."/plugin
+
   import compiler/[ast, idents, modulegraphs, options]
 
   import "."/treesitter/api
 
-const
-  gAtoms* {.used.} = @[
-    "field_identifier",
-    "identifier",
-    "number_literal",
-    "char_literal",
-    "preproc_arg",
-    "primitive_type",
-    "sized_type_specifier",
-    "type_identifier"
-  ].toHashSet()
-
-  gExpressions* {.used.} = @[
-    "parenthesized_expression",
-    "bitwise_expression",
-    "shift_expression",
-    "math_expression",
-    "escape_sequence"
-  ].toHashSet()
-
-  gEnumVals* {.used.} = @[
-    "identifier",
-    "number_literal",
-    "char_literal"
-  ].concat(toSeq(gExpressions.items))
-
 type
-  Kind* = enum
-    exactlyOne
-    oneOrMore     # +
-    zeroOrMore    # *
-    zeroOrOne     # ?
-    orWithNext    # !
-
-  Ast* = object
-    name*: string
-    kind*: Kind
-    recursive*: bool
-    children*: seq[ref Ast]
-    when defined(TOAST):
-      tonim*: proc (ast: ref Ast, node: TSNode, gState: State)
-    regex*: Regex
-
-  AstTable* {.used.} = TableRef[string, seq[ref Ast]]
+  Feature* = enum
+    ast1, ast2
 
   State* = ref object
     # Command line arguments to toast - some forwarded from cimport.nim
@@ -79,31 +40,25 @@ type
     typeMap*: TableRef[string, string]
                                # `--typeMap | -T` to map instances of type X to Y - e.g. ABC=cint
 
-    # cimport.nim specific
-    compile*: seq[string]      # `cCompile()` list of files already processed
-    nocache*: bool             # `cDisableCaching()` to disable caching of artifacts
-    overrides*: string         # `cOverride()` code which gets added to `cPlugin()` output
-    pluginSource*: string      # `cPlugin()` generated code to write to plugin file from
-    searchDirs*: seq[string]   # `cSearchPath()` added directories for header search
-
-    # Data fields
-    code*: string              # Contents of header file currently being processed
-    currentHeader*: string     # Const name of header being currently processed
-    impShort*: string          # Short base name for pragma in output
-    outputHandle*: File        # `--output | -o` open file handle
-    sourceFile*: string        # Full path of header being currently processed
-
-    # Plugin callbacks
-    onSymbol*, onSymbolOverride*: OnSymbol
-    onSymbolOverrideFinal*: OnSymbolOverrideFinal
-
-    # Symbol tables
-    constIdentifiers*: HashSet[string]     # Const names for enum casting
-    identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName
-    skippedSyms*: HashSet[string]          # Symbols that have been skipped due to being unwrappable or
-                                           # the user provided override is blank
-    # Nim compiler objects
     when defined(TOAST):
+      # Data fields
+      code*: string              # Contents of header file currently being processed
+      currentHeader*: string     # Const name of header being currently processed
+      impShort*: string          # Short base name for pragma in output
+      outputHandle*: File        # `--output | -o` open file handle
+      sourceFile*: string        # Full path of header being currently processed
+
+      # Plugin callbacks
+      onSymbol*, onSymbolOverride*: OnSymbol
+      onSymbolOverrideFinal*: OnSymbolOverrideFinal
+
+      # Symbol tables
+      constIdentifiers*: HashSet[string]     # Const names for enum casting
+      identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName
+      skippedSyms*: HashSet[string]          # Symbols that have been skipped due to being unwrappable or
+                                            # the user provided override is blank
+
+      # Nim compiler objects
       constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode
       identCache*: IdentCache
       config*: ConfigRef
@@ -112,37 +67,86 @@ type
       # Table of symbols to generated AST PNode - used to implement forward declarations
       identifierNodes*: TableRef[string, PNode]
 
-    # Used for the exprparser.nim module
-    currentExpr*, currentTyCastName*: string
-    # Controls whether or not the current expression
-    # should validate idents against currently defined idents
-    skipIdentValidation*: bool
+      # Used for the exprparser.nim module
+      currentExpr*, currentTyCastName*: string
+      # Controls whether or not the current expression
+      # should validate idents against currently defined idents
+      skipIdentValidation*: bool
 
-    # Legacy AST fields, remove when ast2 becomes default
-    constStr*, enumStr*, procStr*, typeStr*: string
-    commentStr*, debugStr*, skipStr*: string
-    data*: seq[tuple[name, val: string]]
-    nodeBranch*: seq[string]
+      # Legacy AST fields, remove when ast2 becomes default
+      constStr*, enumStr*, procStr*, typeStr*: string
+      commentStr*, debugStr*, skipStr*: string
+      data*: seq[tuple[name, val: string]]
+      nodeBranch*: seq[string]
+    else:
+      # cimport.nim specific
+      compile*: seq[string]      # `cCompile()` list of files already processed
+      nocache*: bool             # `cDisableCaching()` to disable caching of artifacts
+      overrides*: string         # `cOverride()` code which gets added to `cPlugin()` output
+      pluginSource*: string      # `cPlugin()` generated code to write to plugin file from
+      searchDirs*: seq[string]   # `cSearchPath()` added directories for header search
 
-  Feature* = enum
-    ast1, ast2
+when defined(TOAST):
+  const
+    gAtoms* {.used.} = @[
+      "field_identifier",
+      "identifier",
+      "number_literal",
+      "char_literal",
+      "preproc_arg",
+      "primitive_type",
+      "sized_type_specifier",
+      "type_identifier"
+    ].toHashSet()
 
-var
-  gStateCT* {.compiletime, used.} = new(State)
+    gExpressions* {.used.} = @[
+      "parenthesized_expression",
+      "bitwise_expression",
+      "shift_expression",
+      "math_expression",
+      "escape_sequence"
+    ].toHashSet()
+
+    gEnumVals* {.used.} = @[
+      "identifier",
+      "number_literal",
+      "char_literal"
+    ].concat(toSeq(gExpressions.items))
+
+  type
+    Kind* = enum
+      exactlyOne
+      oneOrMore     # +
+      zeroOrMore    # *
+      zeroOrOne     # ?
+      orWithNext    # !
+
+    Ast* = object
+      name*: string
+      kind*: Kind
+      recursive*: bool
+      children*: seq[ref Ast]
+      tonim*: proc (ast: ref Ast, node: TSNode, gState: State)
+      regex*: Regex
+
+    AstTable* {.used.} = TableRef[string, seq[ref Ast]]
+
+  # Redirect output to file when required
+  template gecho*(args: string) =
+    if gState.outputHandle.isNil:
+      echo args
+    else:
+      gState.outputHandle.writeLine(args)
+
+  template decho*(args: varargs[string, `$`]): untyped =
+    if gState.debug:
+      gecho join(args, "").getCommented()
+else:
+  var
+    gStateCT* {.compiletime, used.} = new(State)
 
 template nBl*(s: typed): untyped {.used.} =
   (s.len != 0)
 
 template Bl*(s: typed): untyped {.used.} =
-  (s.len == 0)
-
-# Redirect output to file when required
-template gecho*(args: string) =
-  if gState.outputHandle.isNil:
-    echo args
-  else:
-    gState.outputHandle.writeLine(args)
-
-template decho*(args: varargs[string, `$`]): untyped =
-  if gState.debug:
-    gecho join(args, "").getCommented()
+  (s.len == 0)

nimterop/toast.nim

@@ -4,8 +4,6 @@ import "."/treesitter/[api, c, cpp]
 
 import "."/[ast, ast2, build, globals, getters, grammar, tshelp]
 
-{.passC: "-DNIMTEROP".}
-
 proc process(gState: State, path: string, astTable: AstTable) =
   doAssert existsFile(path), &"Invalid path {path}"
 

nimterop/tshelp.nim

@@ -79,6 +79,23 @@ proc getStartAtom*(node: TSNode): int =
       else:
         break
 
+proc getConstQualifier*(gState: State, node: TSNode): bool =
+  # Check if node siblings have type_qualifier = `const`
+  var
+    curr = node.tsNodePrevNamedSibling()
+  while not curr.isNil:
+    # Check previous siblings
+    if curr.getName() == "type_qualifier" and
+      gState.getNodeVal(curr) == "const":
+        return true
+    curr = curr.tsNodePrevNamedSibling()
+
+  # Check immediate next sibling
+  curr = node.tsNodePrevNamedSibling()
+  if curr.getName() == "type_qualifier" and
+    gState.getNodeVal(curr) == "const":
+      return true
+
 proc getXCount*(node: TSNode, ntype: string, reverse = false): int =
   if not node.isNil:
     # Get number of ntype nodes nested in tree

tests/tast2.nim

@@ -2,6 +2,8 @@ import macros, os, sets, strutils
 
 import nimterop/[cimport]
 
+{.passC: "-DNIMTEROP".}
+
 static:
   # Skip casting on lower nim compilers because
   # the VM does not support it