merged better html links #850

compiler/commands.nim

@@ -267,6 +267,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "out", "o": 
     expectArg(switch, arg, pass, info)
     options.outFile = arg
+  of "docseesrcurl":
+    expectArg(switch, arg, pass, info)
+    options.docSeeSrcUrl = arg
   of "mainmodule", "m":
     expectArg(switch, arg, pass, info)
     optMainModule = arg

compiler/docgen.nim

@@ -14,7 +14,7 @@
 import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite,
-  importer, sempass2, json
+  importer, sempass2, json, xmltree, cgi, typesrenderer
 
 type
   TSections = array[TSymKind, PRope]
@@ -23,8 +23,9 @@ type
     id: int                  # for generating IDs
     toc, section: TSections
     indexValFilename: string
+    seenSymbols: PStringTable # avoids duplicate symbol generation for HTML.
 
-  PDoc* = ref TDocumentor
+  PDoc* = ref TDocumentor ## Alias to type less.
 
 proc compilerMsgHandler(filename: string, line, col: int,
                         msgKind: rst.TMsgKind, arg: string) {.procvar.} =
@@ -59,6 +60,7 @@ proc newDocumentor*(filename: string, config: PStringTable): PDoc =
   initRstGenerator(result[], (if gCmd != cmdRst2tex: outHtml else: outLatex),
                    options.gConfigVars, filename, {roSupportRawDirective},
                    docgenFindFile, compilerMsgHandler)
+  result.seenSymbols = newStringTable(modeCaseInsensitive)
   result.id = 100
 
 proc dispA(dest: var PRope, xml, tex: string, args: openArray[PRope]) =
@@ -144,6 +146,23 @@ proc genRecComment(d: PDoc, n: PNode): PRope =
   else:
     n.comment = nil
 
+proc getPlainDocstring(n: PNode): string =
+  ## Gets the plain text docstring of a node non destructively.
+  ##
+  ## You need to call this before genRecComment, whose side effects are removal
+  ## of comments from the tree. The proc will recursively scan and return all
+  ## the concatenated ``##`` comments of the node.
+  result = ""
+  if n == nil: return
+  if n.comment != nil and startsWith(n.comment, "##"):
+    result = n.comment
+  if result.len < 1:
+    if n.kind notin {nkEmpty..nkNilLit}:
+      for i in countup(0, len(n)-1):
+        result = getPlainDocstring(n.sons[i])
+        if result.len > 0: return
+
+
 proc findDocComment(n: PNode): PNode =
   if n == nil: return nil
   if not isNil(n.comment) and startsWith(n.comment, "##"): return n
@@ -205,14 +224,111 @@ proc getRstName(n: PNode): PRstNode =
     internalError(n.info, "getRstName()")
     result = nil
 
+proc newUniquePlainSymbol(d: PDoc, original: string): string =
+  ## Returns a new unique plain symbol made up from the original.
+  ##
+  ## When a collision is found in the seenSymbols table, new numerical variants
+  ## with underscore + number will be generated.
+  if not d.seenSymbols.hasKey(original):
+    result = original
+    d.seenSymbols[original] = ""
+    return
+
+  # Iterate over possible numeric variants of the original name.
+  var count = 2
+
+  while true:
+    result = original & "_" & $count
+    if not d.seenSymbols.hasKey(result):
+      d.seenSymbols[result] = ""
+      break
+    count += 1
+
+
+proc complexName(k: TSymKind, n: PNode, baseName: string): string =
+  ## Builds a complex unique href name for the node.
+  ##
+  ## Pass as ``baseName`` the plain symbol obtained from the nodeName. The
+  ## format of the returned symbol will be ``baseName(.callable type)?,(param
+  ## type)?(,param type)*``. The callable type part will be added only if the
+  ## node is not a proc, as those are the common ones. The suffix will be a dot
+  ## and a single letter representing the type of the callable. The parameter
+  ## types will be added with a preceeding dash. Return types won't be added.
+  ##
+  ## If you modify the output of this proc, please update the anchor generation
+  ## section of ``doc/docgen.txt``.
+  result = baseName
+  case k:
+  of skProc: result.add(defaultParamSeparator)
+  of skMacro: result.add(".m" & defaultParamSeparator)
+  of skMethod: result.add(".e" & defaultParamSeparator)
+  of skIterator: result.add(".i" & defaultParamSeparator)
+  of skTemplate: result.add(".t" & defaultParamSeparator)
+  of skConverter: result.add(".c" & defaultParamSeparator)
+  else: discard
+
+  if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
+    result.add(renderParamTypes(n[paramsPos]))
+
+
+proc isCallable(n: PNode): bool =
+  ## Returns true if `n` contains a callable node.
+  case n.kind
+  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
+    nkConverterDef: result = true
+  else:
+    result = false
+
+
+proc docstringSummary(rstText: string): string =
+  ## Returns just the first line or a brief chunk of text from a rst string.
+  ##
+  ## Most docstrings will contain a one liner summary, so stripping at the
+  ## first newline is usually fine. If after that the content is still too big,
+  ## it is stripped at the first comma, colon or dot, usual english sentence
+  ## separators.
+  ##
+  ## No guarantees are made on the size of the output, but it should be small.
+  ## Also, we hope to not break the rst, but maybe we do. If there is any
+  ## trimming done, an ellipsis unicode char is added.
+  const maxDocstringChars = 100
+  assert (rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#'))
+  result = rstText.substr(2).strip
+  var pos = result.find('\L')
+  if pos > 0:
+    result.delete(pos, result.len - 1)
+    result.add("…")
+  if pos < maxDocstringChars:
+    return
+  # Try to keep trimming at other natural boundaries.
+  pos = result.find({'.', ',', ':'})
+  let last = result.len - 1
+  if pos > 0 and pos < last:
+    result.delete(pos, last)
+    result.add("…")
+
+
 proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   if not isVisible(nameNode): return
-  var name = toRope(getName(d, nameNode))
+  let
+    name = getName(d, nameNode)
+    nameRope = name.toRope
+    plainDocstring = getPlainDocstring(n) # call here before genRecComment!
   var result: PRope = nil
-  var literal = ""
+  var literal, plainName = ""
   var kind = tkEof
   var comm = genRecComment(d, n)  # call this here for the side-effect!
   var r: TSrcGen
+  # Obtain the plain rendered string for hyperlink titles.
+  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
+    renderNoPragmas, renderNoProcDefs})
+  while true:
+    getNextTok(r, kind, literal)
+    if kind == tkEof:
+      break
+    plainName.add(literal)
+
+  # Render the HTML hyperlink.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
   while true:
     getNextTok(r, kind, literal)
@@ -253,13 +369,47 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
             [toRope(esc(d.target, literal))])
   inc(d.id)
+  let
+    plainNameRope = toRope(xmltree.escape(plainName.strip))
+    cleanPlainSymbol = renderPlainSymbolName(nameNode)
+    complexSymbol = complexName(k, n, cleanPlainSymbol)
+    plainSymbolRope = toRope(cleanPlainSymbol)
+    plainSymbolEncRope = toRope(URLEncode(cleanPlainSymbol))
+    itemIDRope = toRope(d.id)
+    symbolOrId = d.newUniquePlainSymbol(complexSymbol)
+    symbolOrIdRope = symbolOrId.toRope
+    symbolOrIdEncRope = URLEncode(symbolOrId).toRope
+
+  var seeSrcRope: PRope = nil
+  let docItemSeeSrc = getConfigVar("doc.item.seesrc")
+  if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0:
+    let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl,
+      ["path", "line"], [n.info.toFilename.toRope, toRope($n.info.line)])
+    dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc,
+        ["path", "line", "url"], [n.info.toFilename.toRope,
+        toRope($n.info.line), urlRope])])
+
   app(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"),
-                                        ["name", "header", "desc", "itemID"],
-                                        [name, result, comm, toRope(d.id)]))
+    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"],
+    [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
+      symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope]))
   app(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"),
-                                    ["name", "header", "desc", "itemID"], [
-      toRope(getName(d, nameNode, d.splitAfter)), result, comm, toRope(d.id)]))
-  setIndexTerm(d[], $d.id, getName(d, nameNode))
+    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc"],
+    [toRope(getName(d, nameNode, d.splitAfter)), result, comm,
+      itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope,
+      plainSymbolEncRope, symbolOrIdEncRope]))
+
+  # Ironically for types the complexSymbol is *cleaner* than the plainName
+  # because it doesn't include object fields or documentation comments. So we
+  # use the plain one for callable elements, and the complex for the rest.
+  var linkTitle = changeFileExt(extractFilename(d.filename), "") & " : "
+  if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip))
+  else: linkTitle.add(xmltree.escape(complexSymbol.strip))
+
+  setIndexTerm(d[], symbolOrId, name, linkTitle,
+    xmltree.escape(plainDocstring.docstringSummary))
 
 proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): PJsonNode =
   if not isVisible(nameNode): return

compiler/options.nim

@@ -100,6 +100,8 @@ var
   gSelectedGC* = gcRefc       # the selected GC
   searchPaths*, lazyPaths*: TLinkedList
   outFile*: string = ""
+  docSeeSrcUrl*: string = ""  # if empty, no seeSrc will be generated. \
+  # The string uses the formatting variables `path` and `line`.
   headerFile*: string = ""
   gVerbosity* = 1             # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors

compiler/renderer.nim

@@ -15,7 +15,7 @@ import
 type 
   TRenderFlag* = enum 
     renderNone, renderNoBody, renderNoComments, renderDocComments, 
-    renderNoPragmas, renderIds
+    renderNoPragmas, renderIds, renderNoProcDefs
   TRenderFlags* = set[TRenderFlag]
   TRenderTok*{.final.} = object 
     kind*: TTokType
@@ -51,10 +51,17 @@ proc isKeyword*(s: string): bool =
       (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): 
     result = true
 
-proc renderDefinitionName*(s: PSym): string =
+proc renderDefinitionName*(s: PSym, noQuotes = false): string =
+  ## Returns the definition name of the symbol.
+  ##
+  ## If noQuotes is false the symbol may be returned in backticks. This will
+  ## happen if the name happens to be a keyword or the first character is not
+  ## part of the SymStartChars set.
   let x = s.name.s
-  if x[0] in SymStartChars and not renderer.isKeyword(x): result = x
-  else: result = '`' & x & '`'
+  if noQuotes or (x[0] in SymStartChars and not renderer.isKeyword(x)):
+    result = x
+  else:
+    result = '`' & x & '`'
 
 const 
   IndentWidth = 2
@@ -1119,22 +1126,22 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkStaticStmt: gstaticStmt(g, n)
   of nkAsmStmt: gasm(g, n)
   of nkProcDef: 
-    putWithSpace(g, tkProc, "proc")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkProc, "proc")
     gproc(g, n)
   of nkConverterDef:
-    putWithSpace(g, tkConverter, "converter")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkConverter, "converter")
     gproc(g, n)
   of nkMethodDef: 
-    putWithSpace(g, tkMethod, "method")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkMethod, "method")
     gproc(g, n)
   of nkIteratorDef: 
-    putWithSpace(g, tkIterator, "iterator")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkIterator, "iterator")
     gproc(g, n)
   of nkMacroDef: 
-    putWithSpace(g, tkMacro, "macro")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkMacro, "macro")
     gproc(g, n)
   of nkTemplateDef: 
-    putWithSpace(g, tkTemplate, "template")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkTemplate, "template")
     gproc(g, n)
   of nkTypeSection: 
     gsection(g, n, emptyContext, tkType, "type")
@@ -1336,4 +1343,3 @@ proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) =
     inc(r.idx)
   else: 
     kind = tkEof
-  

compiler/typesrenderer.nim

@@ -0,0 +1,113 @@
+import renderer, strutils, ast, msgs, types
+
+const defaultParamSeparator* = ","
+
+proc renderPlainSymbolName*(n: PNode): string =
+  ## Returns the first non '*' nkIdent node from the tree.
+  ##
+  ## Use this on documentation name nodes to extract the *raw* symbol name,
+  ## without decorations, parameters, or anything. That can be used as the base
+  ## for the HTML hyperlinks.
+  result = ""
+  case n.kind
+  of nkPostfix:
+    for i in 0 .. <n.len:
+      result = renderPlainSymbolName(n[<n.len])
+      if result.len > 0:
+        return
+  of nkIdent:
+    if n.ident.s != "*":
+      result = n.ident.s
+  of nkSym:
+    result = n.sym.renderDefinitionName(noQuotes = true)
+  of nkPragmaExpr:
+    result = renderPlainSymbolName(n[0])
+  of nkAccQuoted:
+    result = renderPlainSymbolName(n[<n.len])
+  else:
+    internalError(n.info, "renderPlainSymbolName() with " & $n.kind)
+  assert (not result.isNil)
+
+proc renderType(n: PNode): string =
+  ## Returns a string with the node type or the empty string.
+  case n.kind:
+  of nkIdent: result = n.ident.s
+  of nkSym: result = typeToString(n.sym.typ)
+  of nkVarTy:
+    assert len(n) == 1
+    result = renderType(n[0])
+  of nkRefTy:
+    assert len(n) == 1
+    result = "ref." & renderType(n[0])
+  of nkPtrTy:
+    assert len(n) == 1
+    result = "ptr." & renderType(n[0])
+  of nkProcTy:
+    assert len(n) > 1
+    let params = n[0]
+    assert params.kind == nkFormalParams
+    assert len(params) > 0
+    result = "proc("
+    for i in 1 .. <len(params): result.add(renderType(params[i]) & ',')
+    result[<len(result)] = ')'
+  of nkIdentDefs:
+    assert len(n) >= 3
+    let typePos = len(n) - 2
+    let typeStr = renderType(n[typePos])
+    result = typeStr
+    for i in 1 .. <typePos:
+      assert n[i].kind == nkIdent
+      result.add(',' & typeStr)
+  of nkTupleTy:
+    assert len(n) > 0
+    result = "tuple["
+    for i in 0 .. <len(n): result.add(renderType(n[i]) & ',')
+    result[<len(result)] = ']'
+  of nkBracketExpr:
+    assert len(n) >= 2
+    result = renderType(n[0]) & '['
+    for i in 1 .. <len(n): result.add(renderType(n[i]) & ',')
+    result[<len(result)] = ']'
+  else: result = ""
+  assert (not result.isNil)
+
+
+proc renderParamTypes(found: var seq[string], n: PNode) =
+  ## Recursive helper, adds to `found` any types, or keeps diving the AST.
+  ##
+  ## The normal `doc` generator doesn't include .typ information, so the
+  ## function won't render types for parameters with default values. The `doc2`
+  ## generator does include the information.
+  case n.kind
+  of nkFormalParams:
+    for i in 1 .. <len(n): renderParamTypes(found, n[i])
+  of nkIdentDefs:
+    # These are parameter names + type + default value node.
+    let typePos = len(n) - 2
+    assert typePos > 0
+    var typeStr = renderType(n[typePos])
+    if typeStr.len < 1:
+      # Try with the last node, maybe its a default value.
+      assert n[typePos+1].kind != nkEmpty
+      let typ = n[typePos+1].typ
+      if not typ.isNil: typeStr = typeToString(typ, preferExported)
+      if typeStr.len < 1:
+        return
+    for i in 0 .. <typePos:
+      assert n[i].kind == nkIdent
+      found.add(typeStr)
+  else:
+    internalError(n.info, "renderParamTypes(found,n) with " & $n.kind)
+
+proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string =
+  ## Returns the types contained in `n` joined by `sep`.
+  ##
+  ## This proc expects to be passed as `n` the parameters of any callable. The
+  ## string output is meant for the HTML renderer. If there are no parameters,
+  ## the empty string is returned. The parameters will be joined by `sep` but
+  ## other characters may appear too, like ``[]`` or ``|``.
+  result = ""
+  var found: seq[string] = @[]
+  renderParamTypes(found, n)
+  if found.len > 0:
+    result = found.join(sep)