Remove nimgen dependency, enable tmath test

.gitignore

@@ -5,3 +5,5 @@ nimcache
 *.exe
 *.swp
 toast
+inc
+tests

.travis.yml

@@ -25,8 +25,7 @@ install:
       curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh
       sh init.sh -y
   - export PATH=$HOME/.nimble/bin:$PATH
-  # - nimble refresh -y
 
 script:
-  - nimble installWithDeps
+  - nimble install
   - nimble test

appveyor.yml

@@ -78,7 +78,7 @@ for:
   - /home/appveyor/binaries
 
 build_script:
-  - nimble installWithDeps
+  - nimble install
 
 test_script:
   - nimble test

nimterop.nimble

@@ -10,7 +10,7 @@ installDirs = @["nimterop"]
 
 # Dependencies
 
-requires "nim >= 0.19.0", "treesitter >= 0.1.0", "treesitter_c >= 0.1.0", "treesitter_cpp >= 0.1.0", "regex >= 0.10.0", "cligen >= 0.9.17"
+requires "nim >= 0.19.0", "regex >= 0.10.0", "cligen >= 0.9.17"
 
 proc execCmd(cmd:string)=
   echo cmd
@@ -20,13 +20,6 @@ task test, "Test":
   execCmd "nim c -r tests/tnimterop_c.nim"
   execCmd "nim cpp -r tests/tnimterop_c.nim"
   execCmd "nim cpp -r tests/tnimterop_cpp.nim"
-  # when defined(windows):
-    # execCmd "nim c -r tests/tmath.nim"
-    # execCmd "nim cpp -r tests/tmath.nim"
-
-task installWithDeps, "install dependencies":
-  for a in ["http://github.com/genotrance/nimtreesitter?subdir=treesitter",
-            "http://github.com/genotrance/nimtreesitter?subdir=treesitter_c",
-            "http://github.com/genotrance/nimtreesitter?subdir=treesitter_cpp",]:
-    execCmd "nimble install -y " & a
-  execCmd "nimble install -y"
+  when defined(windows):
+    execCmd "nim c -r tests/tmath.nim"
+    execCmd "nim cpp -r tests/tmath.nim"

nimterop/ast.nim

@@ -2,9 +2,7 @@ import strformat, strutils, tables
 
 import regex
 
-import treesitter/runtime
-
-import "."/[getters, globals, grammar]
+import "."/[getters, globals, grammar, treesitter/runtime]
 
 const gAtoms = @[
   "field_identifier",

nimterop/cimport.nim

@@ -1,6 +1,6 @@
 import macros, os, strformat, strutils
 
-import getters, globals
+import "."/globals
 
 proc interpPath(dir: string): string=
   # TODO: more robust: needs a DirSep after "$projpath"
@@ -42,6 +42,14 @@ proc getToast(fullpath: string, recurse: bool = false): string =
   doAssert exitCode == 0, $exitCode
   result = output
 
+proc getGccPaths*(mode = "c"): string =
+  var
+    ret = 0
+    nul = when defined(Windows): "nul" else: "/dev/null"
+    mmode = if mode == "cpp": "c++" else: mode
+
+  (result, ret) = gorgeEx("gcc -Wp,-v -x" & mmode & " " & nul)
+
 proc cSearchPath*(path: string): string {.compileTime.}=
   result = findPath(path, fail = false)
   if result.len == 0:

nimterop/getters.nim

@@ -1,12 +1,8 @@
-{.experimental: "codeReordering".}
-
 import macros, os, strformat, strutils, tables
 
 import regex
 
-import treesitter/runtime
-
-import git, globals
+import "."/[git, globals, treesitter/runtime]
 
 const gReserved = """
 addr and as asm
@@ -53,6 +49,17 @@ const gTypeMap = {
 proc sanitizePath*(path: string): string =
   path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("//", $DirSep)])
 
+proc getType*(str: string): string =
+  if str == "void":
+    return "object"
+
+  result = str.strip(chars={'_'}).
+    replace(re"([u]?int[\d]+)_t", "$1").
+    replace(re"([u]?int)ptr_t", "ptr $1")
+
+  if gTypeMap.hasKey(result):
+    result = gTypeMap[result]
+
 proc getIdentifier*(str: string): string =
   result = str.strip(chars={'_'}).replace(re"_+", "_").getType()
 
@@ -69,17 +76,6 @@ proc getUniqueIdentifier*(exists: seq[string], prefix = ""): string =
 
   return name & $count
 
-proc getType*(str: string): string =
-  if str == "void":
-    return "object"
-
-  result = str.strip(chars={'_'}).
-    replace(re"([u]?int[\d]+)_t", "$1").
-    replace(re"([u]?int)ptr_t", "ptr $1")
-
-  if gTypeMap.hasKey(result):
-    result = gTypeMap[result]
-
 proc getPtrType*(str: string): string =
   result = case str:
     of "ptr cchar":
@@ -116,15 +112,7 @@ proc getLineCol*(node: TSNode): tuple[line, col: int] =
 proc getCurrentHeader*(fullpath: string): string =
   ("header" & fullpath.splitFile().name.replace(re"[-.]+", ""))
 
-proc getGccPaths*(mode = "c"): string =
-  var
-    ret = 0
-    nul = when defined(Windows): "nul" else: "/dev/null"
-    mmode = if mode == "cpp": "c++" else: mode
-
-  (result, ret) = gorgeEx("gcc -Wp,-v -x" & mmode & " " & nul)
-
-proc removeStatic*(content: string): string =
+proc removeStatic(content: string): string =
   ## Replace static function bodies with a semicolon and commented
   ## out body
   return content.replace(

nimterop/git.nim

@@ -1,6 +1,6 @@
 import macros, os, osproc, regex, strformat, strutils
 
-import globals
+import "."/globals
 
 proc execAction*(cmd: string, nostderr=false): string =
   var
@@ -89,7 +89,6 @@ macro gitPull*(url: static string, outdirN = "", plistN = "", checkoutN = ""): u
 
     discard execAction(&"cd \"{outdir}\" && git config core.sparsecheckout true")
     writeFile(sparsefile, plist)
-    echo "Wrote"
 
   if checkout.len != 0:
     echo "Checking out " & checkout

nimterop/globals.nim

@@ -2,7 +2,8 @@ import tables
 
 import regex
 
-import treesitter/runtime
+when defined(NIMTEROP):
+  import "."/treesitter/runtime
 
 type
   Kind* = enum
@@ -16,7 +17,8 @@ type
     name*: string
     kind*: Kind
     children*: seq[ref Ast]
-    tonim*: proc (ast: ref Ast, node: TSNode)
+    when defined(NIMTEROP):
+      tonim*: proc (ast: ref Ast, node: TSNode)
     regex*: Regex
 
   State* = object
@@ -31,7 +33,8 @@ type
 
     ast*: Table[string, seq[ref Ast]]
     data*: seq[tuple[name, val: string]]
-    grammar*: seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode) {.nimcall.}]]
+    when defined(NIMTEROP):
+      grammar*: seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode) {.nimcall.}]]
 
 var
   gStateCT* {.compiletime.}: State

nimterop/grammar.nim

@@ -2,9 +2,7 @@ import strformat, strutils, tables
 
 import regex
 
-import treesitter/runtime
-
-import "."/[getters, globals, lisp]
+import "."/[getters, globals, lisp, treesitter/runtime]
 
 proc initGrammar() =
   # #define X Y

nimterop/setup.nim

@@ -0,0 +1,43 @@
+import os, strutils
+
+import "."/git
+
+static:
+  gitPull("https://github.com/tree-sitter/tree-sitter/", "inc/treesitter", """
+include/*
+src/runtime/*
+""")
+
+  gitPull("https://github.com/JuliaStrings/utf8proc", "inc/utf8proc", """
+*.c
+*.h
+""")
+
+  gitPull("https://github.com/tree-sitter/tree-sitter-c", "inc/treesitter_c", """
+src/*.h
+src/*.c
+src/*.cc
+""")
+
+  gitPull("https://github.com/tree-sitter/tree-sitter-cpp", "inc/treesitter_cpp", """
+src/*.h
+src/*.c
+src/*.cc
+""")
+
+  let
+    stack = "inc/treesitter/src/runtime/stack.c"
+    headerc = "inc/treesitter_c/src/parser.h"
+    headercpp = "inc/treesitter_cpp/src/parser.h"
+
+  stack.writeFile(stack.readFile().replace("inline Stack", "Stack"))
+
+  headerc.writeFile("""
+    typedef struct TSLanguage TSLanguage;
+    const TSLanguage *tree_sitter_c();
+  """)
+
+  headercpp.writeFile("""
+    typedef struct TSLanguage TSLanguage;
+    const TSLanguage *tree_sitter_cpp();
+  """)

nimterop/treesitter/c.nim

@@ -0,0 +1,5 @@
+import "."/runtime
+
+{.compile: ("../../inc/treesitter_c/src/parser.c", "parserc.o").}
+
+proc treeSitterC*(): ptr TSLanguage {.importc: "tree_sitter_c", header: "inc/treesitter_c/src/parser.h".}

nimterop/treesitter/cpp.nim

@@ -0,0 +1,6 @@
+import "."/runtime
+
+{.compile: ("../../inc/treesitter_cpp/src/parser.c", "parsercpp.o").}
+{.compile: ("../../inc/treesitter_cpp/src/scanner.cc", "scannercpp.o").}
+
+proc treeSitterCpp*(): ptr TSLanguage {.importc: "tree_sitter_cpp", header: "inc/treesitter_cpp/src/parser.h".}

nimterop/treesitter/runtime.nim

@@ -0,0 +1,132 @@
+{.experimental: "codeReordering".}
+
+{.passC: "-std=c11 -DUTF8PROC_STATIC".}
+{.passC: "-Iinc/treesitter/include".}
+{.passC: "-Iinc/treesitter/src".}
+{.passC: "-Iinc/utf8proc".}
+{.compile: "../../inc/treesitter/src/runtime/runtime.c".}
+
+type TSInputEncoding* = distinct int
+converter enumToInt(en: TSInputEncoding): int {.used.} = en.int
+
+type TSSymbolType* = distinct int
+converter enumToInt(en: TSSymbolType): int {.used.} = en.int
+
+type TSLogType* = distinct int
+converter enumToInt(en: TSLogType): int {.used.} = en.int
+
+const
+  headerruntime = "inc/treesitter/include/tree_sitter/runtime.h"
+  TREE_SITTER_LANGUAGE_VERSION* = 9
+  TSInputEncodingUTF8* = 0.TSInputEncoding
+  TSInputEncodingUTF16* = 1.TSInputEncoding
+  TSSymbolTypeRegular* = 0.TSSymbolType
+  TSSymbolTypeAnonymous* = 1.TSSymbolType
+  TSSymbolTypeAuxiliary* = 2.TSSymbolType
+  TSLogTypeParse* = 0.TSLogType
+  TSLogTypeLex* = 1.TSLogType
+
+type
+  TSSymbol* = uint16
+  TSLanguage* = object
+  TSParser* = object
+  TSTree* = object
+  TSPoint* {.importc: "TSPoint", header: headerruntime, bycopy.} = object
+    row*: uint32
+    column*: uint32
+  TSRange* {.importc: "TSRange", header: headerruntime, bycopy.} = object
+    start_point*: TSPoint
+    end_point*: TSPoint
+    start_byte*: uint32
+    end_byte*: uint32
+  TSInput* {.importc: "TSInput", header: headerruntime, bycopy.} = object
+    payload*: pointer
+    read*: proc(byte_index: uint32,position: TSPoint,bytes_read: ptr uint32): cchar {.nimcall.}
+    encoding*: TSInputEncoding
+  TSLogger* {.importc: "TSLogger", header: headerruntime, bycopy.} = object
+    payload*: pointer
+    log*: proc(a1: TSLogType,a2: cstring) {.nimcall.}
+  TSInputEdit* {.importc: "TSInputEdit", header: headerruntime, bycopy.} = object
+    start_byte*: uint32
+    old_end_byte*: uint32
+    new_end_byte*: uint32
+    start_point*: TSPoint
+    old_end_point*: TSPoint
+    new_end_point*: TSPoint
+  TSNode* {.importc: "TSNode", header: headerruntime, bycopy.} = object
+    context*: array[4, uint32]
+    id*: pointer
+    tree*: ptr TSTree
+  TSTreeCursor* {.importc: "TSTreeCursor", header: headerruntime, bycopy.} = object
+    tree*: pointer
+    id*: pointer
+    context*: array[2, uint32]
+
+proc ts_parser_new*(): ptr TSParser {.importc: "ts_parser_new", header: headerruntime.}
+proc ts_parser_delete*(a1: ptr TSParser) {.importc: "ts_parser_delete", header: headerruntime.}
+proc ts_parser_language*(a1: ptr TSParser): ptr TSLanguage {.importc: "ts_parser_language", header: headerruntime.}
+proc ts_parser_set_language*(a1: ptr TSParser,a2: ptr TSLanguage): bool {.importc: "ts_parser_set_language", header: headerruntime.}
+proc ts_parser_logger*(a1: ptr TSParser): TSLogger {.importc: "ts_parser_logger", header: headerruntime.}
+proc ts_parser_set_logger*(a1: ptr TSParser,a2: TSLogger) {.importc: "ts_parser_set_logger", header: headerruntime.}
+proc ts_parser_print_dot_graphs*(a1: ptr TSParser,a2: ptr FILE) {.importc: "ts_parser_print_dot_graphs", header: headerruntime.}
+proc ts_parser_halt_on_error*(a1: ptr TSParser,a2: bool) {.importc: "ts_parser_halt_on_error", header: headerruntime.}
+proc ts_parser_parse*(a1: ptr TSParser,a2: ptr TSTree,a3: TSInput): ptr TSTree {.importc: "ts_parser_parse", header: headerruntime.}
+proc ts_parser_parse_string*(a1: ptr TSParser,a2: ptr TSTree,a3: cstring,a4: uint32): ptr TSTree {.importc: "ts_parser_parse_string", header: headerruntime.}
+proc ts_parser_parse_string_encoding*(a1: ptr TSParser,a2: ptr TSTree,a3: cstring,a4: uint32,a5: TSInputEncoding): ptr TSTree {.importc: "ts_parser_parse_string_encoding", header: headerruntime.}
+proc ts_parser_enabled*(a1: ptr TSParser): bool {.importc: "ts_parser_enabled", header: headerruntime.}
+proc ts_parser_set_enabled*(a1: ptr TSParser,a2: bool) {.importc: "ts_parser_set_enabled", header: headerruntime.}
+proc ts_parser_operation_limit*(a1: ptr TSParser): uint {.importc: "ts_parser_operation_limit", header: headerruntime.}
+proc ts_parser_set_operation_limit*(a1: ptr TSParser,a2: uint) {.importc: "ts_parser_set_operation_limit", header: headerruntime.}
+proc ts_parser_reset*(a1: ptr TSParser) {.importc: "ts_parser_reset", header: headerruntime.}
+proc ts_parser_set_included_ranges*(a1: ptr TSParser,a2: ptr TSRange,a3: uint32) {.importc: "ts_parser_set_included_ranges", header: headerruntime.}
+proc ts_parser_included_ranges*(a1: ptr TSParser,a2: ptr uint32): ptr TSRange {.importc: "ts_parser_included_ranges", header: headerruntime.}
+proc ts_tree_copy*(a1: ptr TSTree): ptr TSTree {.importc: "ts_tree_copy", header: headerruntime.}
+proc ts_tree_delete*(a1: ptr TSTree) {.importc: "ts_tree_delete", header: headerruntime.}
+proc ts_tree_root_node*(a1: ptr TSTree): TSNode {.importc: "ts_tree_root_node", header: headerruntime.}
+proc ts_tree_edit*(a1: ptr TSTree,a2: ptr TSInputEdit) {.importc: "ts_tree_edit", header: headerruntime.}
+proc ts_tree_get_changed_ranges*(a1: ptr TSTree,a2: ptr TSTree,a3: ptr uint32): ptr TSRange {.importc: "ts_tree_get_changed_ranges", header: headerruntime.}
+proc ts_tree_print_dot_graph*(a1: ptr TSTree,a2: ptr FILE) {.importc: "ts_tree_print_dot_graph", header: headerruntime.}
+proc ts_tree_language*(a1: ptr TSTree): ptr TSLanguage {.importc: "ts_tree_language", header: headerruntime.}
+proc ts_node_start_byte*(a1: TSNode): uint32 {.importc: "ts_node_start_byte", header: headerruntime.}
+proc ts_node_start_point*(a1: TSNode): TSPoint {.importc: "ts_node_start_point", header: headerruntime.}
+proc ts_node_end_byte*(a1: TSNode): uint32 {.importc: "ts_node_end_byte", header: headerruntime.}
+proc ts_node_end_point*(a1: TSNode): TSPoint {.importc: "ts_node_end_point", header: headerruntime.}
+proc ts_node_symbol*(a1: TSNode): TSSymbol {.importc: "ts_node_symbol", header: headerruntime.}
+proc ts_node_type*(a1: TSNode): cstring {.importc: "ts_node_type", header: headerruntime.}
+proc ts_node_string*(a1: TSNode): cstring {.importc: "ts_node_string", header: headerruntime.}
+proc ts_node_eq*(a1: TSNode,a2: TSNode): bool {.importc: "ts_node_eq", header: headerruntime.}
+proc ts_node_is_null*(a1: TSNode): bool {.importc: "ts_node_is_null", header: headerruntime.}
+proc ts_node_is_named*(a1: TSNode): bool {.importc: "ts_node_is_named", header: headerruntime.}
+proc ts_node_is_missing*(a1: TSNode): bool {.importc: "ts_node_is_missing", header: headerruntime.}
+proc ts_node_has_changes*(a1: TSNode): bool {.importc: "ts_node_has_changes", header: headerruntime.}
+proc ts_node_has_error*(a1: TSNode): bool {.importc: "ts_node_has_error", header: headerruntime.}
+proc ts_node_parent*(a1: TSNode): TSNode {.importc: "ts_node_parent", header: headerruntime.}
+proc ts_node_child*(a1: TSNode,a2: uint32): TSNode {.importc: "ts_node_child", header: headerruntime.}
+proc ts_node_named_child*(a1: TSNode,a2: uint32): TSNode {.importc: "ts_node_named_child", header: headerruntime.}
+proc ts_node_child_count*(a1: TSNode): uint32 {.importc: "ts_node_child_count", header: headerruntime.}
+proc ts_node_named_child_count*(a1: TSNode): uint32 {.importc: "ts_node_named_child_count", header: headerruntime.}
+proc ts_node_next_sibling*(a1: TSNode): TSNode {.importc: "ts_node_next_sibling", header: headerruntime.}
+proc ts_node_next_named_sibling*(a1: TSNode): TSNode {.importc: "ts_node_next_named_sibling", header: headerruntime.}
+proc ts_node_prev_sibling*(a1: TSNode): TSNode {.importc: "ts_node_prev_sibling", header: headerruntime.}
+proc ts_node_prev_named_sibling*(a1: TSNode): TSNode {.importc: "ts_node_prev_named_sibling", header: headerruntime.}
+proc ts_node_first_child_for_byte*(a1: TSNode,a2: uint32): TSNode {.importc: "ts_node_first_child_for_byte", header: headerruntime.}
+proc ts_node_first_named_child_for_byte*(a1: TSNode,a2: uint32): TSNode {.importc: "ts_node_first_named_child_for_byte", header: headerruntime.}
+proc ts_node_descendant_for_byte_range*(a1: TSNode,a2: uint32,a3: uint32): TSNode {.importc: "ts_node_descendant_for_byte_range", header: headerruntime.}
+proc ts_node_named_descendant_for_byte_range*(a1: TSNode,a2: uint32,a3: uint32): TSNode {.importc: "ts_node_named_descendant_for_byte_range", header: headerruntime.}
+proc ts_node_descendant_for_point_range*(a1: TSNode,a2: TSPoint,a3: TSPoint): TSNode {.importc: "ts_node_descendant_for_point_range", header: headerruntime.}
+proc ts_node_named_descendant_for_point_range*(a1: TSNode,a2: TSPoint,a3: TSPoint): TSNode {.importc: "ts_node_named_descendant_for_point_range", header: headerruntime.}
+proc ts_node_edit*(a1: ptr TSNode,a2: ptr TSInputEdit) {.importc: "ts_node_edit", header: headerruntime.}
+proc ts_tree_cursor_new*(a1: TSNode): TSTreeCursor {.importc: "ts_tree_cursor_new", header: headerruntime.}
+proc ts_tree_cursor_delete*(a1: ptr TSTreeCursor) {.importc: "ts_tree_cursor_delete", header: headerruntime.}
+proc ts_tree_cursor_reset*(a1: ptr TSTreeCursor,a2: TSNode) {.importc: "ts_tree_cursor_reset", header: headerruntime.}
+proc ts_tree_cursor_current_node*(a1: ptr TSTreeCursor): TSNode {.importc: "ts_tree_cursor_current_node", header: headerruntime.}
+proc ts_tree_cursor_goto_parent*(a1: ptr TSTreeCursor): bool {.importc: "ts_tree_cursor_goto_parent", header: headerruntime.}
+proc ts_tree_cursor_goto_next_sibling*(a1: ptr TSTreeCursor): bool {.importc: "ts_tree_cursor_goto_next_sibling", header: headerruntime.}
+proc ts_tree_cursor_goto_first_child*(a1: ptr TSTreeCursor): bool {.importc: "ts_tree_cursor_goto_first_child", header: headerruntime.}
+proc ts_tree_cursor_goto_first_child_for_byte*(a1: ptr TSTreeCursor,a2: uint32): int64 {.importc: "ts_tree_cursor_goto_first_child_for_byte", header: headerruntime.}
+proc ts_language_symbol_count*(a1: ptr TSLanguage): uint32 {.importc: "ts_language_symbol_count", header: headerruntime.}
+proc ts_language_symbol_name*(a1: ptr TSLanguage,a2: TSSymbol): cstring {.importc: "ts_language_symbol_name", header: headerruntime.}
+proc ts_language_symbol_for_name*(a1: ptr TSLanguage,a2: cstring): TSSymbol {.importc: "ts_language_symbol_for_name", header: headerruntime.}
+proc ts_language_symbol_type*(a1: ptr TSLanguage,a2: TSSymbol): TSSymbolType {.importc: "ts_language_symbol_type", header: headerruntime.}
+proc ts_language_version*(a1: ptr TSLanguage): uint32 {.importc: "ts_language_version", header: headerruntime.}
+

tests/tmath.nim

@@ -3,6 +3,8 @@ import unittest
 
 type
   locale_t = object
+  mingw_ldbl_type_t = object
+  mingw_dbl_type_t = object
 
 cDebug()
 

toast.nim

@@ -1,8 +1,8 @@
+import nimterop/setup
+
 import os, strformat, strutils
 
-import treesitter/runtime
-import treesitter_c/c
-import treesitter_cpp/cpp
+import nimterop/treesitter/[runtime, c, cpp]
 
 import nimterop/[ast, globals, getters, grammar]
 

toast.nims

@@ -0,0 +1 @@
+switch("d", "NIMTEROP")