Merge branch 'override'

nimterop/ast.nim

@@ -1,4 +1,4 @@
-import os, sets, strformat, strutils, tables, times
+import macros, os, sets, strformat, strutils, tables, times
 
 import regex
 
@@ -187,6 +187,7 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable
   if nimState.enumStr.nBl:
     echo &"{nimState.enumStr}\n"
 
+  nimState.constStr = nimState.getOverrideFinal(nskConst) & nimState.constStr
   if nimState.constStr.nBl:
     echo &"const{nimState.constStr}\n"
 
@@ -195,9 +196,11 @@ proc printNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable
 {{.pragma: {nimState.impShort}C, {nimState.impShort}, cdecl{nimState.getDynlib()}.}}
 """
 
+  nimState.typeStr = nimState.getOverrideFinal(nskType) & nimState.typeStr
   if nimState.typeStr.nBl:
     echo &"type{nimState.typeStr}\n"
 
+  nimState.procStr = nimState.getOverrideFinal(nskProc) & nimState.procStr
   if nimState.procStr.nBl:
     echo &"{nimState.procStr}\n"
 

nimterop/cimport.nim

@@ -172,8 +172,8 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
 macro cOverride*(body): untyped =
   ## When the wrapper code generated by nimterop is missing certain symbols or not
   ## accurate, it may be required to hand wrap them. Define them in a
-  ## `cOverride() <cimport.html#cOverride.m>`_ macro block so that Nimterop no
+  ## `cOverride() <cimport.html#cOverride.m>`_ macro block so that Nimterop uses
-  ## longer defines these symbols.
+  ## these definitions instead.
   ##
   ## For example:
   ##
@@ -195,28 +195,70 @@ macro cOverride*(body): untyped =
   ##      proc svGetCallerInfo(fileName: var cstring; lineNumber: var cint)
   ##
   ## Using the `cOverride() <cimport.html#cOverride.m>`_ block, nimterop
-  ## can be instructed to skip over `svGetCallerInfo()`. This works for procs,
+  ## can be instructed to use this definition of `svGetCallerInfo()` instead.
-  ## consts and types.
+  ## This works for procs, consts and types.
   ##
   ## `cOverride() <cimport.html#cOverride.m>`_ only affects calls to
   ## `cImport() <cimport.html#cImport.m%2C%2Cstring%2Cstring%2Cstring>`_ that follow it.
 
-  proc recFindIdent(node: NimNode): seq[string] =
+  iterator findOverrides(node: NimNode): tuple[name, override: string, kind: NimNodeKind] =
-    if node.kind != nnkIdent:
+    for child in node:
-      for child in node:
+      case child.kind
-        result.add recFindIdent(child)
+      of nnkTypeSection, nnkConstSection:
-        if result.len != 0 and node.kind notin [nnkTypeSection, nnkConstSection]:
+        # Types, const
-          break
+        for inst in child:
-    elif $node != "*":
+          let name =
-      result.add $node
+            if inst[0].kind == nnkPragmaExpr:
+              $inst[0][0]
+            else:
+              $inst[0]
 
-  for sym in body:
+          yield (name.strip(chars={'*'}), inst.repr, child.kind)
-    gStateCT.symOverride.add recFindIdent(sym)
+      of nnkProcDef:
+        let
+          name = $child[0]
 
-  result = body
+        yield (name.strip(chars={'*'}), child.repr, child.kind)
+      else:
+        discard
 
-  if gStateCT.debug:
+  if gStateCT.overrides.len == 0:
-    echo "# Overriding " & gStateCT.symOverride.join(" ")
+    gStateCT.overrides = """
+import sets, tables
+
+proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} =
+"""
+
+  # If cPlugin called before cOverride
+  if gStateCT.pluginSourcePath.len != 0:
+    gStateCT.pluginSourcePath = ""
+
+  var
+    names: seq[string]
+  for name, override, kind in body.findOverrides():
+    let
+      typ =
+        case kind
+        of nnkTypeSection: "nskType"
+        of nnkConstSection: "nskConst"
+        of nnkProcDef: "nskProc"
+        else: ""
+
+    gStateCT.overrides &= &"""
+  if sym.name == "{name}" and sym.kind == {typ} and "{name}" in cOverrides["{typ}"]:
+    sym.override = """ & "\"\"\"" & override & "\"\"\"\n"
+
+    gStateCT.overrides &= &"    cOverrides[\"{typ}\"].excl \"{name}\"\n"
+
+    gStateCT.overrides = gStateCT.overrides.replace("proc onSymbolOverride",
+      &"cOverrides[\"{typ}\"].incl \"{name}\"\nproc onSymbolOverride")
+
+    names.add name
+
+    gStateCT.symOverride.add name
+
+  if gStateCT.debug and names.len != 0:
+    echo "# Overriding " & names.join(" ")
 
 proc cSkipSymbol*(skips: seq[string]) {.compileTime.} =
   ## Similar to `cOverride() <cimport.html#cOverride.m>`_, this macro allows
@@ -228,6 +270,22 @@ proc cSkipSymbol*(skips: seq[string]) {.compileTime.} =
     static: cSkipSymbol @["proc1", "Type2"]
   gStateCT.symOverride.add skips
 
+proc cPluginHelper(body: string) =
+  gStateCT.pluginSource = body
+
+  let
+    data = "import macros, nimterop/plugin\n\n" & body & gStateCT.overrides
+    hash = data.hash().abs()
+    path = getProjectCacheDir("cPlugins", forceClean = false) / "nimterop_" & $hash & ".nim"
+
+  if not fileExists(path) or gStateCT.nocache or compileOption("forceBuild"):
+    mkDir(path.parentDir())
+    writeFile(path, data)
+
+  doAssert fileExists(path), "Unable to write plugin file: " & path
+
+  gStateCT.pluginSourcePath = path
+
 macro cPlugin*(body): untyped =
   ## When `cOverride() <cimport.html#cOverride.m>`_ and
   ## `cSkipSymbol() <cimport.html#cSkipSymbol%2Cseq[T][string]>`_
@@ -276,18 +334,7 @@ macro cPlugin*(body): untyped =
         if sym.kind == nskProc and sym.name.contains("SDL_"):
           sym.name = sym.name.replace("SDL_", "")
 
-  let
+  cPluginHelper(body.repr)
-    data = "import macros, nimterop/plugin\n\n" & body.repr
-    hash = data.hash().abs()
-    path = getProjectCacheDir("cPlugins", forceClean = false) / "nimterop_" & $hash & ".nim"
-
-  if not fileExists(path) or gStateCT.nocache or compileOption("forceBuild"):
-    mkDir(path.parentDir())
-    writeFile(path, data)
-
-  doAssert fileExists(path), "Unable to write plugin file: " & path
-
-  gStateCT.pluginSourcePath = path
 
 proc cSearchPath*(path: string): string {.compileTime.}=
   ## Get full path to file or directory `path` in search path configured
@@ -529,6 +576,9 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st
   let
     fullpath = findPath(filename)
 
+  if gStateCT.overrides.len != 0 and gStateCT.pluginSourcePath.len == 0:
+    cPluginHelper(gStateCT.pluginSource)
+
   echo "# Importing " & fullpath.sanitizePath
 
   let

nimterop/getters.nim

@@ -138,18 +138,44 @@ proc getUniqueIdentifier*(nimState: NimState, prefix = ""): string =
 
   return name & $count
 
-proc addNewIdentifer*(nimState: NimState, name: string): bool =
+proc addNewIdentifer*(nimState: NimState, name: string, override = false): bool =
-  if name notin nimState.gState.symOverride:
+  if override or name notin nimState.gState.symOverride:
     let
       nimName = name[0] & name[1 .. ^1].replace("_", "").toLowerAscii
 
     if nimState.identifiers.hasKey(nimName):
-      doAssert name == nimState.identifiers[nimName], &"Identifier '{name}' is a stylistic duplicate of identifier '{nimState.identifiers[nimName]}', use 'cPlugin:onSymbol()' to rename"
+      doAssert name == nimState.identifiers[nimName],
+        &"Identifier '{name}' is a stylistic duplicate of identifier " &
+        &"'{nimState.identifiers[nimName]}', use 'cPlugin:onSymbol()' to rename"
       result = false
     else:
       nimState.identifiers[nimName] = name
       result = true
 
+proc getOverride*(nimState: NimState, name: string, kind: NimSymKind): string =
+  doAssert name.len != 0, "Blank identifier error"
+
+  if nimState.gState.onSymbolOverride != nil:
+    var
+      nname = nimState.getIdentifier(name, kind, "Override")
+      sym = Symbol(name: nname, kind: kind)
+    if nname.nBl:
+      nimState.gState.onSymbolOverride(sym)
+
+      if sym.override.len != 0 and nimState.addNewIdentifer(nname, override = true):
+        result = sym.override
+
+        if kind != nskProc:
+          result = result.replace(re"(?m)^(.*?)$", "  $1")
+
+proc getOverrideFinal*(nimState: NimState, kind: NimSymKind): string =
+  let
+    typ = $kind
+
+  if nimState.gState.onSymbolOverrideFinal != nil:
+    for i in nimState.gState.onSymbolOverrideFinal(typ):
+      result &= "\n" & nimState.getOverride(i, kind)
+
 proc getPtrType*(str: string): string =
   result = case str:
     of "ptr cchar":
@@ -320,6 +346,9 @@ proc getAstChildByName*(ast: ref Ast, name: string): ref Ast =
     if name in ast.children[i].name.split("|"):
       return ast.children[i]
 
+  if ast.children.len == 1 and ast.children[0].name == ".":
+    return ast.children[0]
+
 proc getPxName*(node: TSNode, offset: int): string =
   var
     np = node
@@ -449,7 +478,10 @@ proc loadPlugin*(gState: State, sourcePath: string) =
   doAssert lib != nil, "Plugin $1 compiled to $2 failed to load" % [sourcePath, pdll]
 
   gState.onSymbol = cast[OnSymbol](lib.symAddr("onSymbol"))
-  doAssert gState.onSymbol != nil, "onSymbol() load failed from " & pdll
+
+  gState.onSymbolOverride = cast[OnSymbol](lib.symAddr("onSymbolOverride"))
+
+  gState.onSymbolOverrideFinal = cast[OnSymbolOverrideFinal](lib.symAddr("onSymbolOverrideFinal"))
 
 proc expandSymlinkAbs*(path: string): string =
   try:

nimterop/globals.nim

@@ -57,9 +57,10 @@ type
 
     nocache*, nocomments*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool
 
-    code*, dynlib*, mode*, nim*, pluginSourcePath*: string
+    code*, dynlib*, mode*, nim*, overrides*, pluginSource*, pluginSourcePath*: string
 
-    onSymbol*: OnSymbol
+    onSymbol*, onSymbolOverride*: OnSymbol
+    onSymbolOverrideFinal*: OnSymbolOverrideFinal
 
   NimState {.used.} = ref object
     identifiers*: TableRef[string, string]

nimterop/grammar.nim

@@ -20,14 +20,19 @@ proc initGrammar(): Grammar =
         nimState.debugStr &= "\n# define X Y"
 
       let
+        name = nimState.data[0].val
+        nname = nimState.getIdentifier(name, nskConst)
         val = nimState.data[1].val.getLit()
 
-      if val.nBl:
+      if not nname.nBl:
         let
-          name = nimState.getIdentifier(nimState.data[0].val, nskConst)
+          override = nimState.getOverride(name, nskConst)
-
+        if override.len != 0:
-        if name.nBl and nimState.addNewIdentifer(name):
+          nimState.constStr &= &"{nimState.getComments()}\n{override}"
-          nimState.constStr &= &"{nimState.getComments()}\n  {name}* = {val}"
+        else:
+          nimState.constStr &= &"{nimState.getComments()}\n  # Const '{name}' skipped"
+      elif val.nBl and nimState.addNewIdentifer(nname):
+        nimState.constStr &= &"{nimState.getComments()}\n  {nname}* = {val}"
   ))
 
   let
@@ -175,7 +180,12 @@ proc initGrammar(): Grammar =
       let
         pragma = nimState.getPragma(pragmas)
 
-      if nname notin gTypeMap and typ.nBl and nname.nBl and nimState.addNewIdentifer(nname):
+      if not nname.nBl:
+        let
+          override = nimState.getOverride(name, nskType)
+        if override.len != 0:
+          nimState.typeStr &= &"{nimState.getComments()}\n{override}"
+      elif nname notin gTypeMap and typ.nBl and nimState.addNewIdentifer(nname):
         if i < nimState.data.len and nimState.data[^1].name == "function_declarator":
           var
             fname = nname
@@ -266,7 +276,12 @@ proc initGrammar(): Grammar =
                   union = ", union"
               break
 
-    if nname.nBl and nimState.addNewIdentifer(nname):
+    if not nname.nBl:
+      let
+        override = nimState.getOverride(name, nskType)
+      if override.len != 0:
+        nimState.typeStr &= &"{nimState.getComments()}\n{override}"
+    elif nimState.addNewIdentifer(nname):
       if nimState.data.len == 1:
         nimState.typeStr &= &"{nimState.getComments()}\n  {nname}* {{.bycopy{union}.}} = object"
       else:
@@ -604,7 +619,12 @@ proc initGrammar(): Grammar =
         if pout.len != 0 and pout[^2 .. ^1] == ", ":
           pout = pout[0 .. ^3]
 
-        if fnname.nBl and nimState.addNewIdentifer(fnname):
+        if not fnname.nBl:
+          let
+            override = nimState.getOverride(fname, nskProc)
+          if override.len != 0:
+            nimState.typeStr &= &"{nimState.getComments()}\n{override}"
+        elif nimState.addNewIdentifer(fnname):
           let
             ftyp = nimState.getIdentifier(nimState.data[0].val, nskType, fnname).getType()
             pragma = nimState.getPragma(nimState.getImportC(fname, fnname), "cdecl")
@@ -637,6 +657,38 @@ proc initGrammar(): Grammar =
         nimState.commentStr &= &"\n  # {line.strip(leading=false)}"
   ))
 
+  # // unknown
+  result.add((&"""
+   (type_definition|struct_specifier|union_specifier|enum_specifier|declaration
+    (^.*)
+   )
+  """,
+    proc (ast: ref Ast, node: TSNode, nimState: NimState) =
+      for i in nimState.data:
+        case $node.tsNodeType()
+        of "declaration":
+          if i.name == "identifier":
+            let
+              override = nimState.getOverride(i.val, nskProc)
+
+            if override.len != 0:
+              nimState.procStr &= &"{nimState.getComments(true)}\n{override}"
+              break
+            else:
+              nimState.procStr &= &"{nimState.getComments(true)}\n# Declaration '{i.val}' skipped"
+
+        else:
+          if i.name == "type_identifier":
+            let
+              override = nimState.getOverride(i.val, nskType)
+
+            if override.len != 0:
+              nimState.typeStr &= &"{nimState.getComments()}\n{override}"
+            else:
+              nimState.typeStr &= &"{nimState.getComments()}\n  # Type '{i.val}' skipped"
+
+  ))
+
 proc initRegex(ast: ref Ast) =
   if ast.children.len != 0:
     if not ast.recursive:
@@ -653,7 +705,7 @@ proc initRegex(ast: ref Ast) =
       raise newException(Exception, getCurrentExceptionMsg())
 
 proc parseGrammar*(): AstTable =
-  let grammars = initGrammar()
+  const grammars = initGrammar()
 
   result = newTable[string, seq[ref Ast]]()
   for i in 0 .. grammars.len-1:

nimterop/plugin.nim

@@ -1,9 +1,22 @@
-import macros
+import macros, sets, tables
 
 type
   Symbol* = object
     name*: string
     parent*: string
     kind*: NimSymKind
+    override*: string
 
   OnSymbol* = proc(sym: var Symbol) {.cdecl.}
+  OnSymbolOverrideFinal* = proc(typ: string): HashSet[string] {.cdecl.}
+
+var
+  cOverrides*: Table[string, HashSet[string]]
+
+cOverrides = initTable[string, HashSet[string]]()
+cOverrides["nskType"] = initSet[string]()
+cOverrides["nskConst"] = initSet[string]()
+cOverrides["nskProc"] = initSet[string]()
+
+proc onSymbolOverrideFinal*(typ: string): HashSet[string] {.exportc, dynlib.} =
+  result = cOverrides[typ]

tests/include/test.c

@@ -75,3 +75,11 @@ void *multiline2(void) {
 }
 
 void multiline3(void) {}
+
+int weirdfunc(char ***apple) {
+  return 1;
+}
+
+int weirdfunc2(char **apple) {
+  return 2;
+}

tests/include/test.h

@@ -167,6 +167,31 @@ enum
     TDEFL_BOGUS_3 = TDEFL_OUT_BUF_SIZE / TDEFL_BOGUS_1
 };
 
+// cOverride
+struct foo { int foo[8][1]; };
+
+typedef struct tagBITMAPINFOHEADER{
+          int biClrImportant;
+} BITMAPINFOHEADER, * pBITMAPINFOHEADER;
+
+#define BIT 123u
+
+#define BIT2 123u
+
+#define BIT3 123
+
+typedef int ABC;
+
+typedef int DEF;
+
+typedef struct {
+  int **f1;
+} GHI;
+
+struct JKL {
+  int **f1;
+};
+
 #ifdef __cplusplus
 }
 #endif

tests/tmath.nim

@@ -20,7 +20,7 @@ cPlugin:
   import strutils
 
   proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
-    sym.name = sym.name.strip(chars={'_'})
+    sym.name = sym.name.strip(chars={'_'}).replace("__", "_")
 
 cImport cSearchPath("math.h")
 

tests/tnimterop_c.nim

@@ -20,6 +20,27 @@ cPlugin:
     else:
       sym.name = sym.name.strip(chars={'_'})
 
+cOverride:
+  type
+    BITMAPINFOHEADER* {.bycopy.} = object
+      biClrImportant*: int
+
+    Random = object
+
+    ABC = pointer
+
+    GHI = object
+      f2: ptr ptr cint
+
+    JKL = object
+      f2: ptr ptr cint
+
+  const
+    BIT* = 1
+
+  proc weirdfunc(apple: ptr ptr ptr cchar): int {.importc.}
+  proc weirdfunc2(mango: ptr ptr cchar): int {.importc.}
+
 cImport cSearchPath "test.h"
 
 check TEST_INT == 512
@@ -183,3 +204,23 @@ var
   arr: array[5, cint]
 
 check test_array_param(arr) == nil
+
+# cOverride
+
+var
+  ca = weirdfunc
+  cb: BITMAPINFOHEADER
+  cc = weirdfunc2
+  cd: ABC
+  ce: DEF
+  cf: GHI
+  cg: JKL
+
+cd = nil
+ce = 5
+cf.f2 = nil
+cg.f2 = nil
+
+doAssert BIT == 1
+doAssert ca(nil) == 1
+doAssert cc(nil) == 2

tests/zlib.nim

@@ -49,7 +49,8 @@ cOverride:
     alloc_func* {.importc.} = proc(opaque: voidpf, items, size: uint) {.cdecl.}
     Bytef {.importc.} = object
 
-  when defined(posix):
+when defined(posix):
+  cOverride:
     type
       pthread_mutex_s = object
       pthread_cond_s = object