Fix missing comment in ast2

Fix paths import

Skip docgen for rsa for now

appveyor.yml

@@ -83,7 +83,7 @@ for:
   - /home/appveyor/binaries
 
 build_script:
-  - nimble --verbose install -y
+  - nimble --verbose develop -y
 
 test_script:
   - nimble --verbose test

nimterop.nimble

@@ -12,14 +12,21 @@ installDirs = @["nimterop"]
 requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45"
 
 import nimterop/docs
+import os
 
 proc execCmd(cmd: string) =
   exec "tests/timeit " & cmd
 
-proc execTest(test: string, flags = "") =
+proc execTest(test: string, flags = "", runDocs = true) =
   execCmd "nim c --hints:off -f " & flags & " -r " & test
   execCmd "nim cpp --hints:off " & flags & " -r " & test
 
+  if runDocs:
+    let docPath = "build/html_" & test.extractFileName.changeFileExt("") & "_docs"
+    rmDir docPath
+    mkDir docPath
+    buildDocs(@[test], docPath, nimArgs = flags)
+
 task buildToast, "build toast":
   execCmd("nim c --hints:off nimterop/toast.nim")
 

nimterop/ast2.nim

@@ -691,6 +691,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
         let
           fdecl = node[i].anyChildInTree("field_declaration_list")
           edecl = node[i].anyChildInTree("enumerator_list")
+          commentNodes = gState.getCommentNodes(node[i])
 
           # `tname` is name of nested struct / union / enum just
           # added, passed on as type name for field in `newIdentDefs()`
@@ -716,6 +717,7 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode =
         # Add nkIdentDefs for each field
         for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true):
           if not field.isNil:
+            field.comment = gState.getCommentsStr(commentNodes)
             result.add field
 
 proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) =
@@ -725,6 +727,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
   # If `fname` is set, use it as the name when creating new PNode
   # If `istype` is set, this is a typedef, else struct/union
   decho("addTypeObject()")
+  let commentNodes = gState.getCommentNodes(node.tsNodeParent())
+
   let
     # Object has fields or not
     fdlist = node.anyChildInTree("field_declaration_list")
@@ -837,6 +841,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
         gState.addPragma(node, typeDef[0][1], pragmas)
 
     # nkTypeSection.add
+    typeDef.comment = gState.getCommentsStr(commentNodes)
     gState.typeSection.add typeDef
 
     gState.printDebug(typeDef)
@@ -848,6 +853,7 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
       # Current node has fields
       let
         origname = gState.getNodeVal(node.getAtom())
+        commentNodes = gState.getCommentNodes(node)
 
         # Fix issue #185
         name =
@@ -859,6 +865,8 @@ proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = ""
       if name.nBl and gState.identifierNodes.hasKey(name):
         let
           def = gState.identifierNodes[name]
+        def.comment = gState.getCommentsStr(commentNodes)
+
         # Duplicate nkTypeDef for `name` with empty fields
         if def.kind == nkTypeDef and def.len == 3 and
           def[2].kind == nkObjectTy and def[2].len == 3 and
@@ -890,6 +898,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) =
   decho("addTypeTyped()")
   let
     start = getStartAtom(node)
+    commentNodes = gState.getCommentNodes(node)
   for i in start+1+offset ..< node.len:
     # Add a type of a specific type
     let
@@ -897,6 +906,7 @@ proc addTypeTyped(gState: State, node: TSNode, ftname = "", offset = 0) =
       typeDef = gState.newXIdent(node[i], istype = true)
 
     if not typeDef.isNil:
+      typeDef.comment = gState.getCommentsStr(commentNodes)
       let
         name = typeDef.getIdentName()
 
@@ -1007,6 +1017,7 @@ proc addTypeArray(gState: State, node: TSNode) =
     # node[start] = identifier = type name
     (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray")
     tident = gState.getIdent(tname, info, exported = false)
+    commentNodes = gState.getCommentNodes(node)
 
   # Could have multiple types, comma separated
   for i in start+1 ..< node.len:
@@ -1040,6 +1051,7 @@ proc addTypeArray(gState: State, node: TSNode) =
       #  )
       # )
 
+      typeDef.comment = gState.getCommentsStr(commentNodes)
       # nkTypeSection.add
       gState.typeSection.add typeDef
 
@@ -1386,21 +1398,33 @@ proc addEnum(gState: State, node: TSNode) =
         gState.typeSection.add eoverride
     elif gState.addNewIdentifer(name):
       # Add enum definition and helpers
-      gState.enumSection.add gState.parseString(&"defineEnum({name})")
+      let defineNode = gState.parseString(&"defineEnum({name})")
+      # nkStmtList(
+      #  nkCall(
+      #   nkIdent("defineEnum"),
+      #   nkIdent(name) <- set the comment here
+      #  )
+      # )
+      defineNode[0][1].comment = gState.getCommentsStr(gState.getCommentNodes(node))
+      gState.enumSection.add defineNode
 
       # Create const for fields
       var
         fnames: HashSet[string]
         # Hold all of field information so that we can add all of them
         # after the const identifiers has been updated
-        fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode]]]
+        fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]]
       for i in 0 .. enumlist.len - 1:
         let
           en = enumlist[i]
         if en.getName() == "comment":
           continue
+
         let
-          fname = gState.getIdentifier(gState.getNodeVal(en.getAtom()), nskEnumField)
+          atom = en.getAtom()
+          commentNodes = gState.getCommentNodes(en)
+          fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField)
+
         if fname.nBl and gState.addNewIdentifer(fname):
           var
             fval = ""
@@ -1412,9 +1436,9 @@ proc addEnum(gState: State, node: TSNode) =
             fval = &"({prev} + 1).{name}"
 
           if en.len > 1 and en[1].getName() in gEnumVals:
-            fieldDeclarations.add((fname, "", some(en[1])))
+            fieldDeclarations.add((fname, "", some(en[1]), commentNodes))
           else:
-            fieldDeclarations.add((fname, fval, none(TSNode)))
+            fieldDeclarations.add((fname, fval, none(TSNode), commentNodes))
 
           fnames.incl fname
           prev = fname
@@ -1424,18 +1448,20 @@ proc addEnum(gState: State, node: TSNode) =
       gState.constIdentifiers.incl fnames
 
       # parseCExpression requires all const identifiers to be present for the enum
-      for (fname, fval, cexprNode) in fieldDeclarations:
+      for (fname, fval, cexprNode, commentNodes) in fieldDeclarations:
         var fval = fval
         if cexprNode.isSome:
           fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name
         # Cannot use newConstDef() since parseString(fval) adds backticks to and/or
-        gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0]
+        let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0]
+        constNode.comment = gState.getCommentsStr(commentNodes)
+        gState.constSection.add constNode
 
       # Add other names
       if node.getName() == "type_definition" and node.len > 1:
         gState.addTypeTyped(node, ftname = name, offset = offset)
 
-proc addProcVar(gState: State, node, rnode: TSNode) =
+proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) =
   # Add a proc variable
   decho("addProcVar()")
   let
@@ -1488,12 +1514,13 @@ proc addProcVar(gState: State, node, rnode: TSNode) =
     #  nkEmpty()
     # )
 
+    identDefs.comment = gState.getCommentsStr(commentNodes)
     # nkVarSection.add
     gState.varSection.add identDefs
 
     gState.printDebug(identDefs)
 
-proc addProc(gState: State, node, rnode: TSNode) =
+proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) =
   # Add a proc
   #
   # `node` is the `nth` child of (declaration)
@@ -1599,6 +1626,8 @@ proc addProc(gState: State, node, rnode: TSNode) =
     procDef.add newNode(nkEmpty)
     procDef.add newNode(nkEmpty)
 
+    procDef.comment = gState.getCommentsStr(commentNodes)
+
     # nkProcSection.add
     gState.procSection.add procDef
 
@@ -1612,15 +1641,33 @@ proc addDecl(gState: State, node: TSNode) =
   let
     start = getStartAtom(node)
 
+  var
+    firstDecl = true
+    commentNodes: seq[TSNode]
+
   for i in start+1 ..< node.len:
     if not node[i].firstChildInTree("function_declarator").isNil:
       # Proc declaration - var or actual proc
       if node[i].getAtom().getPxName(1) == "pointer_declarator":
         # proc var
-        gState.addProcVar(node[i], node[start])
+        if firstDecl:
+          # If it's the first declaration, use the whole node
+          # to get the comment above/below
+          commentNodes = gState.getCommentNodes(node)
+          firstDecl = false
+        else:
+          commentNodes = gState.getCommentNodes(node[i])
+        gState.addProcVar(node[i], node[start], commentNodes)
       else:
         # proc
-        gState.addProc(node[i], node[start])
+        if firstDecl:
+          # If it's the first declaration, use the whole node
+          # to get the comment above/below
+          commentNodes = gState.getCommentNodes(node)
+          firstDecl = false
+        else:
+          commentNodes = gState.getCommentNodes(node[i])
+        gState.addProc(node[i], node[start], commentNodes)
     else:
       # Regular var
       discard
@@ -1632,11 +1679,14 @@ proc addDef(gState: State, node: TSNode) =
   # and will fail at link time
   decho("addDef()")
   gState.printDebug(node)
+
   let
     start = getStartAtom(node)
+    commentNodes = gState.getCommentNodes(node)
+
   if node[start+1].getName() == "function_declarator":
     if gState.isIncludeHeader():
-      gState.addProc(node[start+1], node[start])
+      gState.addProc(node[start+1], node[start], commentNodes)
     else:
       gecho &"\n# proc '$1' skipped - static inline procs require 'includeHeader'" %
         gState.getNodeVal(node[start+1].getAtom())

nimterop/cimport.nim

@@ -434,7 +434,7 @@ proc cAddSearchDir*(dir: string) {.compileTime.} =
   ## Add directory `dir` to the search path used in calls to
   ## `cSearchPath() <cimport.html#cSearchPath,string>`_.
   runnableExamples:
-    import paths, os
+    import nimterop/paths, os
     static:
       cAddSearchDir testsIncludeDir()
     doAssert cSearchPath("test.h").existsFile

nimterop/docs.nim

@@ -36,7 +36,7 @@ proc execAction(cmd: string): string =
   doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result
 
 proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep,
-                defines: openArray[string] = @[]) =
+                defines: openArray[string] = @[], nimArgs = "") =
   ## Generate docs for all specified nim `files` to the specified `path`
   ##
   ## `baseDir` is the project path by default and `files` and `path` are relative
@@ -45,6 +45,8 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath
   ## `defines` is a list of `-d:xxx` define flags (the `xxx` part) that should be passed
   ## to `nim doc` so that `getHeader()` is invoked correctly.
   ##
+  ## `nimArgs` is a string representing extra arguments to send to the `nim doc` call.
+  ##
   ## Use the `--publish` flag with nimble to publish docs contained in
   ## `path` to Github in the `gh-pages` branch. This requires the ghp-import
   ## package for Python: `pip install ghp-import`
@@ -70,7 +72,7 @@ proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath
         defStr
       nim = getCurrentCompilerExe()
     for file in files:
-      echo execAction(&"{nim} doc {defStr} -o:{path} --project --index:on {baseDir & file}")
+      echo execAction(&"{nim} doc {defStr} {nimArgs} -o:{path} --project --index:on {baseDir & file}")
 
     echo execAction(&"{nim} buildIndex -o:{path}/theindex.html {path}")
     when declared(getNimRootDir):

nimterop/getters.nim

@@ -387,6 +387,16 @@ proc getLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
 proc getLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
   getLineCol(gState.code, node)
 
+proc getEndLineCol*(code: var string, node: TSNode): tuple[line, col: int] =
+  # Get line number and column info for node
+  let
+    point = node.tsNodeEndPoint()
+  result.line = point.row.int + 1
+  result.col = point.column.int + 1
+
+proc getEndLineCol*(gState: State, node: TSNode): tuple[line, col: int] =
+  getEndLineCol(gState.code, node)
+
 proc getTSNodeNamedChildCountSansComments*(node: TSNode): int =
   for i in 0 ..< node.len:
     if node.getName() != "comment":
@@ -571,30 +581,30 @@ proc getPreprocessor*(gState: State, fullpath: string): string =
 
   # Include content only from file
   for line in execAction(cmd).output.splitLines():
-    if line.strip() != "":
+    # We want to keep blank lines here for comment processing
-      if line.len > 1 and line[0 .. 1] == "# ":
+    if line.len > 1 and line[0 .. 1] == "# ":
-        start = false
+      start = false
+      let
+        saniLine = line.sanitizePath(noQuote = true)
+      if sfile in saniLine:
+        start = true
+      elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line:
+        start = true
+      elif gState.recurse:
         let
-          saniLine = line.sanitizePath(noQuote = true)
+          pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true)
-        if sfile in saniLine:
+        if pDir.Bl or pDir in saniLine:
           start = true
-        elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line:
+        else:
-          start = true
+          for inc in gState.includeDirs:
-        elif gState.recurse:
+            if inc.absolutePath().sanitizePath(noQuote = true) in saniLine:
-          let
+              start = true
-            pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true)
+              break
-          if pDir.Bl or pDir in saniLine:
+    else:
-            start = true
+      if start:
-          else:
+        if "#undef" in line:
-            for inc in gState.includeDirs:
+          continue
-              if inc.absolutePath().sanitizePath(noQuote = true) in saniLine:
+        rdata.add line
-                start = true
-                break
-      else:
-        if start:
-          if "#undef" in line:
-            continue
-          rdata.add line
   return rdata.join("\n")
 
 converter toString*(kind: Kind): string =
@@ -634,6 +644,109 @@ proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool
   if result.kind != exactlyOne:
     result.name = result.name[0 .. ^2]
 
+proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string =
+  ## Generate a comment from a set of comment nodes. Comment is guaranteed
+  ## to be able to be rendered using nim doc
+  if commentNodes.len > 0:
+    result = "::"
+    for commentNode in commentNodes:
+      result &= "\n  " & gState.getNodeVal(commentNode).strip()
+
+    result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "")
+    result = result.multiReplace([("\n", "\n  "), ("`", "")]).strip()
+
+proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] =
+  ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch``
+  ## nodes before and after the current node
+  ##
+  ## Priority is (closest line number) > comment before > comment after.
+  ## This priority might need to be changed based on the project, but
+  ## for now it is good enough
+
+  # Skip this if we don't want comments
+  if gState.nocomments:
+    return
+
+  let (line, _) = gState.getLineCol(node)
+
+  # Keep track of both directions from a node
+  var
+    prevSibling = node.tsNodePrevNamedSibling()
+    nextSibling = node.tsNodeNextNamedSibling()
+    nilNode: TSNode
+
+  var
+    i = 0
+    prevSiblingDistance, nextSiblingDistance: int = int.high
+    lowestDistance: int
+    commentsFound = false
+
+  while not commentsFound and i < maxSearch:
+    # Distance from the current node will tell us approximately if the
+    # comment belongs to the node. The closer it is in terms of line
+    # numbers, the more we can be sure it's the comment we want
+    if not prevSibling.isNil:
+      if prevSibling.getName() == "comment":
+        prevSiblingDistance = abs(gState.getEndLineCol(prevSibling)[0] - line)
+      else:
+        prevSiblingDistance = int.high
+    if not nextSibling.isNil:
+      if nextSibling.getName() == "comment":
+        nextSiblingDistance = abs(gState.getLineCol(nextSibling)[0] - line)
+      else:
+        nextSiblingDistance = int.high
+
+    lowestDistance = min(prevSiblingDistance, nextSiblingDistance)
+
+    if prevSiblingDistance > maxSearch:
+      # If the line is out of range, skip searching
+      prevSibling = nilNode # Can't do `= nil`
+
+    if nextSiblingDistance > maxSearch:
+      # If the line is out of range, skip searching
+      nextSibling = nilNode
+
+    # Search above the current line for comments. When one is found
+    # keep going to retrieve successive comments for cases with multiple
+    # `//` style comments
+    while (
+      not prevSibling.isNil and
+      prevSibling.getName() == "comment" and
+      prevSiblingDistance == lowestDistance
+    ):
+      # Put the previous nodes in reverse order so the comments
+      # make logical sense
+      result.insert(prevSibling, 0)
+      prevSibling = prevSibling.tsNodePrevNamedSibling()
+      commentsFound = true
+
+    # If we've already found comments above the current line, quit
+    if commentsFound:
+      break
+
+    # Search below or at the current line for comments. When one is found
+    # keep going to retrieve successive comments for cases with multiple
+    # `//` style comments
+    while (
+      not nextSibling.isNil and
+      nextSibling.getName() == "comment" and
+      nextSiblingDistance == lowestDistance
+    ):
+      result.add(nextSibling)
+      nextSibling = nextSibling.tsNodeNextNamedSibling()
+      commentsFound = true
+
+    if commentsFound:
+      break
+
+    # Go to next sibling pair
+    if not prevSibling.isNil:
+      prevSibling = prevSibling.tsNodePrevNamedSibling()
+    if not nextSibling.isNil:
+      nextSibling = nextSibling.tsNodeNextNamedSibling()
+
+    i += 1
+
 proc getTSNodeNamedChildNames*(node: TSNode): seq[string] =
   if node.tsNodeNamedChildCount() != 0:
     for i in 0 .. node.tsNodeNamedChildCount()-1:

tests/tnimterop_c.nim

@@ -1,4 +1,6 @@
 import std/unittest
+import os
+import macros
 import nimterop/cimport
 import nimterop/paths
 
@@ -10,7 +12,7 @@ cDefine("FORCE")
 cIncludeDir testsIncludeDir()
 cCompile cSearchPath("test.c")
 
-cPluginPath("tests/tnimterop_c_plugin.nim")
+cPluginPath(getProjectPath() / "tnimterop_c_plugin.nim")
 
 cOverride:
   type