diff --git a/appveyor.yml b/appveyor.yml index 5275844..d67b01d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ for: - /home/appveyor/binaries build_script: - - nimble --verbose install -y + - nimble --verbose develop -y test_script: - nimble --verbose test diff --git a/nimterop.nimble b/nimterop.nimble index a6ece7c..702c714 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -12,14 +12,21 @@ installDirs = @["nimterop"] requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" import nimterop/docs +import os proc execCmd(cmd: string) = exec "tests/timeit " & cmd -proc execTest(test: string, flags = "") = +proc execTest(test: string, flags = "", runDocs = true) = execCmd "nim c --hints:off -f " & flags & " -r " & test execCmd "nim cpp --hints:off " & flags & " -r " & test + if runDocs: + let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs" + rmDir docPath + mkDir docPath + buildDocs(@[test], docPath, nimArgs = flags) + task buildToast, "build toast": execCmd("nim c --hints:off nimterop/toast.nim") diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 4d609cb..fd70b46 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -691,6 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = let fdecl = node[i].anyChildInTree("field_declaration_list") edecl = node[i].anyChildInTree("enumerator_list") + commentNodes = gState.getCommentNodes(node[i]) # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` @@ -716,6 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Add nkIdentDefs for each field for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): if not field.isNil: + field.comment = gState.getCommentsStr(commentNodes) result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = @@ -725,6 +727,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # If `fname` is set, use it as the name when creating new PNode # If `istype` is set, this is a typedef, else struct/union decho("addTypeObject()") + let commentNodes = gState.getCommentNodes(node.tsNodeParent()) + let # Object has fields or not fdlist = node.anyChildInTree("field_declaration_list") @@ -837,6 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" gState.addPragma(node, typeDef[0][1], pragmas) # nkTypeSection.add + typeDef.comment = gState.getCommentsStr(commentNodes) gState.typeSection.add typeDef gState.printDebug(typeDef) @@ -848,6 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" # Current node has fields let origname = gState.getNodeVal(node.getAtom()) + commentNodes = gState.getCommentNodes(node) # Fix issue #185 name = @@ -859,6 +865,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "" if name.nBl and gState.identifierNodes.hasKey(name): let def = gState.identifierNodes[name] + def.comment = gState.getCommentsStr(commentNodes) + # Duplicate nkTypeDef for `name` with empty fields if def.kind == nkTypeDef and def.len == 3 and def[2].kind == nkObjectTy and def[2].len == 3 and @@ -890,6 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = decho("addTypeTyped()") let start = getStartAtom(node) + commentNodes = gState.getCommentNodes(node) for i in start+1+offset ..< node.len: # Add a type of a specific type let @@ -897,6 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) = typeDef = gState.newXIdent(node[i], istype = true) if not typeDef.isNil: + typeDef.comment = gState.getCommentsStr(commentNodes) let name = typeDef.getIdentName() @@ -1007,6 +1017,7 @@ proc addTypeArray(gState: State, node: TSNode) = # node[start] = identifier = type name (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") tident = gState.getIdent(tname, info, exported = false) + commentNodes = gState.getCommentNodes(node) # Could have multiple types, comma separated for i in start+1 ..< node.len: @@ -1040,6 +1051,7 @@ proc addTypeArray(gState: State, node: TSNode) = # ) # ) + typeDef.comment = gState.getCommentsStr(commentNodes) # nkTypeSection.add gState.typeSection.add typeDef @@ -1386,21 +1398,33 @@ proc addEnum(gState: State, node: TSNode) = gState.typeSection.add eoverride elif gState.addNewIdentifer(name): # Add enum definition and helpers - gState.enumSection.add gState.parseString(&"defineEnum({name})") + let defineNode = gState.parseString(&"defineEnum({name})") + # nkStmtList( + # nkCall( + # nkIdent("defineEnum"), + # nkIdent(name) <- set the comment here + # ) + # ) + defineNode[0][1].comment = gState.getCommentsStr(gState.getCommentNodes(node)) + gState.enumSection.add defineNode # Create const for fields var fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]] + fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] if en.getName() == "comment": continue + let - fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField) + atom = en.getAtom() + commentNodes = gState.getCommentNodes(en) + fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) + if fname.nBl and gState.addNewIdentifer(fname): var fval = "" @@ -1412,9 +1436,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]))) + fieldDeclarations.add((fname, "", some(en[1]), commentNodes)) else: - fieldDeclarations.add((fname, fval, none(TSNode))) + fieldDeclarations.add((fname, fval, none(TSNode), commentNodes)) fnames.incl fname prev = fname @@ -1424,18 +1448,20 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode) in fieldDeclarations: + for (fname, fval, cexprNode, commentNodes) in fieldDeclarations: var fval = fval if cexprNode.isSome: fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0] + let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + constNode.comment = gState.getCommentsStr(commentNodes) + gState.constSection.add constNode # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode) = +proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc variable decho("addProcVar()") let @@ -1488,12 +1514,13 @@ proc addProcVar(gState: State, node, rnode: TSNode) = # nkEmpty() # ) + identDefs.comment = gState.getCommentsStr(commentNodes) # nkVarSection.add gState.varSection.add identDefs gState.printDebug(identDefs) -proc addProc(gState: State, node, rnode: TSNode) = +proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # # `node` is the `nth` child of (declaration) @@ -1599,6 +1626,8 @@ proc addProc(gState: State, node, rnode: TSNode) = procDef.add newNode(nkEmpty) procDef.add newNode(nkEmpty) + procDef.comment = gState.getCommentsStr(commentNodes) + # nkProcSection.add gState.procSection.add procDef @@ -1612,15 +1641,33 @@ proc addDecl(gState: State, node: TSNode) = let start = getStartAtom(node) + var + firstDecl = true + commentNodes: seq[TSNode] + for i in start+1 ..< node.len: if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - gState.addProcVar(node[i], node[start]) + if firstDecl: + # If it's the first declaration, use the whole node + # to get the comment above/below + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) + gState.addProcVar(node[i], node[start], commentNodes) else: # proc - gState.addProc(node[i], node[start]) + if firstDecl: + # If it's the first declaration, use the whole node + # to get the comment above/below + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) + gState.addProc(node[i], node[start], commentNodes) else: # Regular var discard @@ -1632,11 +1679,14 @@ proc addDef(gState: State, node: TSNode) = # and will fail at link time decho("addDef()") gState.printDebug(node) + let start = getStartAtom(node) + commentNodes = gState.getCommentNodes(node) + if node[start+1].getName() == "function_declarator": if gState.isIncludeHeader(): - gState.addProc(node[start+1], node[start]) + gState.addProc(node[start+1], node[start], commentNodes) else: gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" % gState.getNodeVal(node[start+1].getAtom()) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index e081633..99be1f5 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -434,7 +434,7 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to ## `cSearchPath() `_. runnableExamples: - import paths, os + import nimterop/paths, os static: cAddSearchDir testsIncludeDir() doAssert cSearchPath("test.h").existsFile diff --git a/nimterop/docs.nim b/nimterop/docs.nim index d808535..d5bd50c 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -36,7 +36,7 @@ proc execAction(cmd: string): string = doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep, - defines: openArray[string] = @[]) = + defines: openArray[string] = @[], nimArgs = "") = ## Generate docs for all specified nim `files` to the specified `path` ## ## `baseDir` is the project path by default and `files` and `path` are relative @@ -45,6 +45,8 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath ## `defines` is a list of `-d:xxx` define flags (the `xxx` part) that should be passed ## to `nim doc` so that `getHeader()` is invoked correctly. ## + ## `nimArgs` is a string representing extra arguments to send to the `nim doc` call. + ## ## Use the `--publish` flag with nimble to publish docs contained in ## `path` to Github in the `gh-pages` branch. This requires the ghp-import ## package for Python: `pip install ghp-import` @@ -70,7 +72,7 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath defStr nim = getCurrentCompilerExe() for file in files: - echo execAction(&"{nim} doc {defStr} -o:{path} --project --index:on {baseDir & file}") + echo execAction(&"{nim} doc {defStr} {nimArgs} -o:{path} --project --index:on {baseDir & file}") echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}") when declared(getNimRootDir): diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 620c1ec..10c3b5e 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -387,6 +387,16 @@ proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] = proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] = getLineCol(gState.code, node) +proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] = + # Get line number and column info for node + let + point = node.tsNodeEndPoint() + result.line = point.row.int + 1 + result.col = point.column.int + 1 + +proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] = + getEndLineCol(gState.code, node) + proc getTSNodeNamedChildCountSansComments*(node: TSNode): int = for i in 0 ..< node.len: if node.getName() != "comment": @@ -571,30 +581,30 @@ proc getPreprocessor*(gState: State, fullpath: string): string = # Include content only from file for line in execAction(cmd).output.splitLines(): - if line.strip() != "": - if line.len > 1 and line[0 .. 1] == "# ": - start = false + # We want to keep blank lines here for comment processing + if line.len > 1 and line[0 .. 1] == "# ": + start = false + let + saniLine = line.sanitizePath(noQuote = true) + if sfile in saniLine: + start = true + elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + start = true + elif gState.recurse: let - saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine: + pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) + if pDir.Bl or pDir in saniLine: start = true - elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: - start = true - elif gState.recurse: - let - pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) - if pDir.Bl or pDir in saniLine: - start = true - else: - for inc in gState.includeDirs: - if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: - start = true - break - else: - if start: - if "#undef" in line: - continue - rdata.add line + else: + for inc in gState.includeDirs: + if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: + start = true + break + else: + if start: + if "#undef" in line: + continue + rdata.add line return rdata.join("\n") converter toString*(kind: Kind): string = @@ -634,6 +644,109 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool if result.kind != exactlyOne: result.name = result.name[0 .. ^2] +proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = + ## Generate a comment from a set of comment nodes. Comment is guaranteed + ## to be able to be rendered using nim doc + if commentNodes.len > 0: + result = "::" + for commentNode in commentNodes: + result &= "\n " & gState.getNodeVal(commentNode).strip() + + result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") + result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() + +proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = + ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` + ## nodes before and after the current node + ## + ## Priority is (closest line number) > comment before > comment after. + ## This priority might need to be changed based on the project, but + ## for now it is good enough + + # Skip this if we don't want comments + if gState.nocomments: + return + + let (line, _) = gState.getLineCol(node) + + # Keep track of both directions from a node + var + prevSibling = node.tsNodePrevNamedSibling() + nextSibling = node.tsNodeNextNamedSibling() + nilNode: TSNode + + var + i = 0 + prevSiblingDistance, nextSiblingDistance: int = int.high + lowestDistance: int + commentsFound = false + + while not commentsFound and i < maxSearch: + # Distance from the current node will tell us approximately if the + # comment belongs to the node. The closer it is in terms of line + # numbers, the more we can be sure it's the comment we want + if not prevSibling.isNil: + if prevSibling.getName() == "comment": + prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line) + else: + prevSiblingDistance = int.high + if not nextSibling.isNil: + if nextSibling.getName() == "comment": + nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line) + else: + nextSiblingDistance = int.high + + lowestDistance = min(prevSiblingDistance, nextSiblingDistance) + + if prevSiblingDistance > maxSearch: + # If the line is out of range, skip searching + prevSibling = nilNode # Can't do `= nil` + + if nextSiblingDistance > maxSearch: + # If the line is out of range, skip searching + nextSibling = nilNode + + # Search above the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments + while ( + not prevSibling.isNil and + prevSibling.getName() == "comment" and + prevSiblingDistance == lowestDistance + ): + # Put the previous nodes in reverse order so the comments + # make logical sense + result.insert(prevSibling, 0) + prevSibling = prevSibling.tsNodePrevNamedSibling() + commentsFound = true + + # If we've already found comments above the current line, quit + if commentsFound: + break + + # Search below or at the current line for comments. When one is found + # keep going to retrieve successive comments for cases with multiple + # `//` style comments + while ( + not nextSibling.isNil and + nextSibling.getName() == "comment" and + nextSiblingDistance == lowestDistance + ): + result.add(nextSibling) + nextSibling = nextSibling.tsNodeNextNamedSibling() + commentsFound = true + + if commentsFound: + break + + # Go to next sibling pair + if not prevSibling.isNil: + prevSibling = prevSibling.tsNodePrevNamedSibling() + if not nextSibling.isNil: + nextSibling = nextSibling.tsNodeNextNamedSibling() + + i += 1 + proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if node.tsNodeNamedChildCount() != 0: for i in 0 .. node.tsNodeNamedChildCount()-1: diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index 20c1678..ef814ea 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -1,4 +1,6 @@ import std/unittest +import os +import macros import nimterop/cimport import nimterop/paths @@ -10,7 +12,7 @@ cDefine("FORCE") cIncludeDir testsIncludeDir() cCompile cSearchPath("test.c") -cPluginPath("tests/tnimterop_c_plugin.nim") +cPluginPath(getProjectPath() / "tnimterop_c_plugin.nim") cOverride: type