jyapayne/nimterop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'toast-1'

Browse source
This commit is contained in:
Ganesh Viswanathan 2019-12-20 18:16:52 -06:00
commit c388ddde6b
7 changed files with 215 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

31
README.md
View file

@ -93,23 +93,28 @@ The `toast` binary can also be used directly on the CLI:
toast -h
Usage:
main [optional-params] C/C++ source/header
Options(opt-arg sep :|=|spc):
Options(opt-arg sep :|=|spc):
-h, --help print this cligen-erated help
--help-syntax advanced: prepend, multi-val,..
-p, --preprocess bool false run preprocessor on header
-a, --past bool false print AST output
-n, --pnim bool false print Nim output
-r, --recurse bool false process #include files
-c, --nocomments bool false exclude top-level comments from output
-D=, --defines= strings {} definitions to pass to preprocessor
-I=, --includeDirs= strings {} include directory to pass to preprocessor
-l=, --dynlib= string "" Import symbols from library in specified Nim string
-O=, --symOverride= strings {} skip generating specified symbols
--nim= string "nim" use a particular Nim executable (default: $PATH/nim)
--pluginSourcePath= string "" Nim file to build and load as a plugin
--help-syntax advanced: prepend,plurals,..
-k, --check bool false check generated wrapper with compiler
-d, --debug bool false enable debug output
-D=, --defines= strings {} definitions to pass to preprocessor
-l=, --dynlib= string "" Import symbols from library in specified Nim string
-I=, --includeDirs= strings {} include directory to pass to preprocessor
-m=, --mode= string "cpp" language parser: c or cpp
--nim= string "nim" use a particular Nim executable (default: $PATH/nim)
-c, --nocomments bool false exclude top-level comments from output
-o=, --output= string "" file to output content - default stdout
-a, --past bool false print AST output
-g, --pgrammar bool false print grammar
--pluginSourcePath= string "" Nim file to build and load as a plugin
-n, --pnim bool false print Nim output
-E=, --prefix= strings {} Strip prefix from identifiers
-p, --preprocess bool false run preprocessor on header
-r, --recurse bool false process #include files
-s, --stub bool false stub out undefined type references as objects
-F=, --suffix= strings {} Strip suffix from identifiers
-O=, --symOverride= strings {} skip generating specified symbols
```
__Implementation Details__

41
nimterop/cimport.nim
View file

@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1
include "."/globals
import "."/[build, paths, types]
import "."/[build, compat, paths, types]
export types
proc interpPath(dir: string): string=
@ -116,12 +116,7 @@ proc getNimCheckError(output: string): tuple[tmpFile, errors: string] =
doAssert fileExists(result.tmpFile), "Failed to write to cache dir: " & result.tmpFile
let
nim =
when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
getCurrentCompilerExe()
else:
"nim"
(check, _) = gorgeEx(&"{nim} check {result.tmpFile.sanitizePath}")
(check, _) = gorgeEx(&"{getCurrentCompilerExe()} check {result.tmpFile.sanitizePath}")
result.errors = "\n\n" & check
@ -140,7 +135,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
cmd.add " --recurse"
if flags.nBl:
cmd.add flags
cmd.add " " & flags
for i in gStateCT.defines:
cmd.add &" --defines+={i.quoteShell}"
@ -157,8 +152,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
if gStateCT.symOverride.nBl:
cmd.add &" --symOverride={gStateCT.symOverride.join(\",\")}"
when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}"
cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}"
if gStateCT.pluginSourcePath.nBl:
cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}"
@ -300,13 +294,18 @@ macro cPlugin*(body): untyped =
##
## proc onSymbol(sym: var Symbol) {.exportc, dynlib.}
##
## `onSymbol()` can be used to handle symbol name modifications required due to invalid
## characters like leading/trailing `_` or rename symbols that would clash due to Nim's style
## insensitivity. It can also be used to remove prefixes and suffixes like `SDL_`. The symbol
## name and type is provided to the callback and the name can be modified.
## `onSymbol()` can be used to handle symbol name modifications required due
## to invalid characters in identifiers or to rename symbols that would clash
## due to Nim's style insensitivity. The symbol name and type is provided to
## the callback and the name can be modified.
##
## Returning a blank name will result in the symbol being skipped. This will fail for `nskParam`
## and `nskField` since the generated Nim code will be wrong.
## While `cPlugin` can easily remove leading/trailing `_` or prefixes and
## suffixes like `SDL_`, passing `--prefix` or `--suffix` flags to `cImport`
## in the `flags` parameter is much easier. However, these flags will only be
## considered when no `cPlugin` is specified.
##
## Returning a blank name will result in the symbol being skipped. This will
## fail for `nskParam` and `nskField` since the generated Nim code will be wrong.
##
## Symbol types can be any of the following:
## - `nskConst` for constants
@ -316,10 +315,12 @@ macro cPlugin*(body): untyped =
## - `nskEnumField` for enum (field) names, though they are in the global namespace as `nskConst`
## - `nskProc` - for proc names
##
## `nimterop/plugins` is implicitly imported to provide access to standard plugin facilities.
## `nimterop/plugins` is implicitly imported to provide access to standard
## plugin facilities.
##
## `cPlugin() <cimport.html#cPlugin.m>`_ only affects calls to
## `cImport() <cimport.html#cImport.m%2C%2Cstring%2Cstring%2Cstring>`_ that follow it.
## `cImport() <cimport.html#cImport.m%2C%2Cstring%2Cstring%2Cstring>`_ that
## follow it.
runnableExamples:
cPlugin:
import strutils
@ -572,7 +573,9 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st
## `mode` is purely for forward compatibility when toast adds C++ support. It can
## be ignored for the foreseeable future.
##
## `flags` can be used to pass any other command line arguments to `toast`.
## `flags` can be used to pass any other command line arguments to `toast`. A
## good example would be `--prefix` and `--suffix` which strip leading and
## trailing strings from identifiers, `_` being quite common.
##
## `cImport()` consumes and resets preceding `cOverride()` calls. `cPlugin()`
## is retained for the next `cImport()` call unless a new `cPlugin()` call is

2
nimterop/compat.nim
View file

@ -30,3 +30,5 @@ else:
if not base.endsWith DirSep: base.add DirSep
doAssert file.startsWith base
result = file[base.len .. ^1]
proc getCurrentCompilerExe*(): string = "nim"

13
nimterop/getters.nim
View file

@ -112,6 +112,7 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent="
if name notin nimState.gState.symOverride or parent.nBl:
if nimState.gState.onSymbol != nil:
# Use onSymbol from plugin provided by user
var
sym = Symbol(name: name, parent: parent, kind: kind)
nimState.gState.onSymbol(sym)
@ -120,11 +121,23 @@ proc getIdentifier*(nimState: NimState, name: string, kind: NimSymKind, parent="
else:
result = name
# Strip out --prefix from CLI if specified
for str in nimState.gState.prefix:
if result.startsWith(str):
result = result[str.len .. ^1]
# Strip out --suffix from CLI if specified
for str in nimState.gState.suffix:
if result.endsWith(str):
result = result[0 .. ^(str.len+1)]
checkIdentifier(result, $kind, parent, name)
if result in gReserved or (result == "object" and kind != nskType):
# Enclose in backticks since Nim reserved word
result = &"`{result}`"
else:
# Skip identifier since in symOverride
result = ""
proc getUniqueIdentifier*(nimState: NimState, prefix = ""): string =

2
nimterop/globals.nim
View file

@ -53,7 +53,7 @@ type
AstTable {.used.} = TableRef[string, seq[ref Ast]]
State = ref object
compile*, defines*, headers*, includeDirs*, searchDirs*, symOverride*: seq[string]
compile*, defines*, headers*, includeDirs*, searchDirs*, prefix*, suffix*, symOverride*: seq[string]
nocache*, nocomments*, debug*, past*, preprocess*, pnim*, pretty*, recurse*: bool

205
nimterop/toast.nim
View file

@ -1,8 +1,8 @@
import os, strformat, strutils
import os, osproc, strformat, strutils, times
import "."/treesitter/[api, c, cpp]
import "."/[ast, globals, getters, grammar]
import "."/[ast, compat, globals, getters, grammar]
proc printLisp(gState: State, root: TSNode) =
var
@ -54,7 +54,7 @@ proc printLisp(gState: State, root: TSNode) =
break
proc process(gState: State, path: string, astTable: AstTable) =
doAssert existsFile(path), "Invalid path " & path
doAssert existsFile(path), &"Invalid path {path}"
var
parser = tsParserNew()
@ -81,7 +81,7 @@ proc process(gState: State, path: string, astTable: AstTable) =
elif gState.mode == "cpp":
doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
else:
doAssert false, "Invalid parser " & gState.mode
doAssert false, &"Invalid parser {gState.mode}"
var
tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32)
@ -97,84 +97,191 @@ proc process(gState: State, path: string, astTable: AstTable) =
elif gState.preprocess:
echo gState.code
# CLI processing with default values
proc main(
preprocess = false,
past = false,
pnim = false,
recurse = false,
nocomments = false,
defines: seq[string] = @[],
includeDirs: seq[string] = @[],
dynlib: string = "",
symOverride: seq[string] = @[],
nim: string = "nim",
pluginSourcePath: string = "",
check = false,
debug = false,
defines: seq[string] = @[],
dynlib: string = "",
includeDirs: seq[string] = @[],
mode = modeDefault,
nim: string = "nim",
nocomments = false,
output = "",
past = false,
pgrammar = false,
pluginSourcePath: string = "",
pnim = false,
prefix: seq[string] = @[],
preprocess = false,
recurse = false,
stub = false,
suffix: seq[string] = @[],
symOverride: seq[string] = @[],
source: seq[string]
) =
# Setup global state with arguments
var gState = State(
preprocess: preprocess,
past: past,
pnim: pnim,
recurse: recurse,
nocomments: nocomments,
defines: defines,
includeDirs: includeDirs,
dynlib: dynlib,
symOverride: symOverride,
nim: nim,
pluginSourcePath: pluginSourcePath,
debug: debug,
defines: defines,
dynlib: dynlib,
includeDirs: includeDirs,
mode: mode,
pretty: true
nim: nim,
nocomments: nocomments,
past: past,
pluginSourcePath: pluginSourcePath,
pnim: pnim,
prefix: prefix,
preprocess: preprocess,
pretty: true,
recurse: recurse,
suffix: suffix,
symOverride: symOverride
)
# Split some arguments with ,
gState.symOverride = gState.symOverride.getSplitComma()
gState.prefix = gState.prefix.getSplitComma()
gState.suffix = gState.suffix.getSplitComma()
if pluginSourcePath.nBl:
gState.loadPlugin(pluginSourcePath)
# Backup stdout
var
outputFile = output
outputHandle: File
stdoutBackup = stdout
check = check or stub
# Fix output file extention
if outputFile.len != 0:
if outputFile.splitFile().ext != ".nim":
outputFile = outputFile & ".nim"
# Check needs a file
if check and outputFile.len == 0:
outputFile = getTempDir() / "toast_" & ($getTime().toUnix()).addFileExt("nim")
when defined(windows):
# https://github.com/nim-lang/Nim/issues/12939
echo &"Cannot print wrapper with check on Windows, review {outputFile}\n"
# Redirect output to file
if outputFile.len != 0:
when defined(windows):
doAssert stdout.reopen(outputFile, fmWrite), &"Failed to write to {outputFile}"
else:
doAssert outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}"
stdout = outputHandle
# Process grammar into AST
let
astTable = parseGrammar()
if pgrammar:
# Print AST of grammar
astTable.printGrammar()
elif source.nBl:
# Print source after preprocess or Nim output
if gState.pnim:
printNimHeader()
for src in source:
gState.process(src.expandSymlinkAbs(), astTable)
when not defined(windows):
# Restore stdout
stdout = stdoutBackup
# Check Nim output
if gState.pnim and check:
# Run nim check on generated wrapper
var
(check, err) = execCmdEx(&"{getCurrentCompilerExe()} check {outputFile}")
if err != 0:
# Failed check so try stubbing
if stub:
# Close output file to prepend stubs
when not defined(windows):
outputHandle.close()
else:
stdout.close()
# Find undeclared identifiers in error
var
data = ""
stubData = ""
for line in check.splitLines:
if "undeclared identifier" in line:
try:
# Add stub of object type
stubData &= " " & line.split("'")[1] & " = object\n"
except:
discard
# Include in wrapper file
data = outputFile.readFile()
let
idx = data.find("\ntype\n")
if idx != -1:
# In first existing type block
data = data[0 ..< idx+6] & stubData & data[idx+6 .. ^1]
else:
# At the top if none already
data = "type\n" & stubData & data
outputFile.writeFile(data)
# Rerun nim check on stubbed wrapper
(check, err) = execCmdEx(&"{getCurrentCompilerExe()} check {outputFile}")
doAssert err == 0, "# Nim check with stub failed:\n\n" & check
else:
doAssert err == 0, "# Nim check failed:\n\n" & check
when not defined(windows):
# Print wrapper if temporarily redirected to file
if check and output.len == 0:
stdout.write outputFile.readFile()
when isMainModule:
# Setup cligen command line help and short flags
import cligen
dispatch(main, help = {
"preprocess": "run preprocessor on header",
"past": "print AST output",
"pnim": "print Nim output",
"recurse": "process #include files",
"nocomments": "exclude top-level comments from output",
"defines": "definitions to pass to preprocessor",
"includeDirs": "include directory to pass to preprocessor",
"dynlib": "Import symbols from library in specified Nim string",
"symOverride": "skip generating specified symbols",
"nim": "use a particular Nim executable (default: $PATH/nim)",
"pluginSourcePath": "Nim file to build and load as a plugin",
"check": "check generated wrapper with compiler",
"debug": "enable debug output",
"defines": "definitions to pass to preprocessor",
"dynlib": "Import symbols from library in specified Nim string",
"includeDirs": "include directory to pass to preprocessor",
"mode": "language parser: c or cpp",
"nim": "use a particular Nim executable (default: $PATH/nim)",
"nocomments": "exclude top-level comments from output",
"output": "file to output content - default stdout",
"past": "print AST output",
"pgrammar": "print grammar",
"source" : "C/C++ source/header"
"pluginSourcePath": "Nim file to build and load as a plugin",
"pnim": "print Nim output",
"preprocess": "run preprocessor on header",
"recurse": "process #include files",
"source" : "C/C++ source/header",
"prefix": "Strip prefix from identifiers",
"stub": "stub out undefined type references as objects",
"suffix": "Strip suffix from identifiers",
"symOverride": "skip generating specified symbols"
}, short = {
"preprocess": 'p',
"past": 'a',
"pnim": 'n',
"recurse": 'r',
"nocomments": 'c',
"defines": 'D',
"includeDirs": 'I',
"dynlib": 'l',
"symOverride": 'O',
"check": 'k',
"debug": 'd',
"pgrammar": 'g'
"defines": 'D',
"dynlib": 'l',
"includeDirs": 'I',
"nocomments": 'c',
"output": 'o',
"past": 'a',
"pgrammar": 'g',
"pnim": 'n',
"prefix": 'E',
"preprocess": 'p',
"recurse": 'r',
"stub": 's',
"suffix": 'F',
"symOverride": 'O'
})

11
tests/lzma.nim
View file

@ -4,6 +4,7 @@ import nimterop/[build, cimport]
const
baseDir = getProjectCacheDir("nimterop" / "tests" / "liblzma")
flags = "--prefix=___,__,_ --suffix=__,_"
static:
cDebug()
@ -21,12 +22,6 @@ getHeader(
conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo"
)
cPlugin:
import strutils
proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} =
sym.name = sym.name.strip(chars = {'_'})
cOverride:
type
lzma_internal = object
@ -39,8 +34,8 @@ cOverride:
lzma_index_iter = object
when not lzmaStatic:
cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath")
cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath", flags = flags)
else:
cImport(lzmaPath, recurse = true)
cImport(lzmaPath, recurse = true, flags = flags)
echo "liblzma version = " & $lzma_version_string()