merged better html links #850

compiler/commands.nim

@@ -267,6 +267,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "out", "o": 
     expectArg(switch, arg, pass, info)
     options.outFile = arg
+  of "docseesrcurl":
+    expectArg(switch, arg, pass, info)
+    options.docSeeSrcUrl = arg
   of "mainmodule", "m":
     expectArg(switch, arg, pass, info)
     optMainModule = arg

compiler/docgen.nim

@@ -14,7 +14,7 @@
 import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
   wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite,
-  importer, sempass2, json
+  importer, sempass2, json, xmltree, cgi, typesrenderer
 
 type
   TSections = array[TSymKind, PRope]
@@ -23,8 +23,9 @@ type
     id: int                  # for generating IDs
     toc, section: TSections
     indexValFilename: string
+    seenSymbols: PStringTable # avoids duplicate symbol generation for HTML.
 
-  PDoc* = ref TDocumentor
+  PDoc* = ref TDocumentor ## Alias to type less.
 
 proc compilerMsgHandler(filename: string, line, col: int,
                         msgKind: rst.TMsgKind, arg: string) {.procvar.} =
@@ -59,6 +60,7 @@ proc newDocumentor*(filename: string, config: PStringTable): PDoc =
   initRstGenerator(result[], (if gCmd != cmdRst2tex: outHtml else: outLatex),
                    options.gConfigVars, filename, {roSupportRawDirective},
                    docgenFindFile, compilerMsgHandler)
+  result.seenSymbols = newStringTable(modeCaseInsensitive)
   result.id = 100
 
 proc dispA(dest: var PRope, xml, tex: string, args: openArray[PRope]) =
@@ -144,6 +146,23 @@ proc genRecComment(d: PDoc, n: PNode): PRope =
   else:
     n.comment = nil
 
+proc getPlainDocstring(n: PNode): string =
+  ## Gets the plain text docstring of a node non destructively.
+  ##
+  ## You need to call this before genRecComment, whose side effects are removal
+  ## of comments from the tree. The proc will recursively scan and return all
+  ## the concatenated ``##`` comments of the node.
+  result = ""
+  if n == nil: return
+  if n.comment != nil and startsWith(n.comment, "##"):
+    result = n.comment
+  if result.len < 1:
+    if n.kind notin {nkEmpty..nkNilLit}:
+      for i in countup(0, len(n)-1):
+        result = getPlainDocstring(n.sons[i])
+        if result.len > 0: return
+
+
 proc findDocComment(n: PNode): PNode =
   if n == nil: return nil
   if not isNil(n.comment) and startsWith(n.comment, "##"): return n
@@ -205,14 +224,111 @@ proc getRstName(n: PNode): PRstNode =
     internalError(n.info, "getRstName()")
     result = nil
 
+proc newUniquePlainSymbol(d: PDoc, original: string): string =
+  ## Returns a new unique plain symbol made up from the original.
+  ##
+  ## When a collision is found in the seenSymbols table, new numerical variants
+  ## with underscore + number will be generated.
+  if not d.seenSymbols.hasKey(original):
+    result = original
+    d.seenSymbols[original] = ""
+    return
+
+  # Iterate over possible numeric variants of the original name.
+  var count = 2
+
+  while true:
+    result = original & "_" & $count
+    if not d.seenSymbols.hasKey(result):
+      d.seenSymbols[result] = ""
+      break
+    count += 1
+
+
+proc complexName(k: TSymKind, n: PNode, baseName: string): string =
+  ## Builds a complex unique href name for the node.
+  ##
+  ## Pass as ``baseName`` the plain symbol obtained from the nodeName. The
+  ## format of the returned symbol will be ``baseName(.callable type)?,(param
+  ## type)?(,param type)*``. The callable type part will be added only if the
+  ## node is not a proc, as those are the common ones. The suffix will be a dot
+  ## and a single letter representing the type of the callable. The parameter
+  ## types will be added with a preceeding dash. Return types won't be added.
+  ##
+  ## If you modify the output of this proc, please update the anchor generation
+  ## section of ``doc/docgen.txt``.
+  result = baseName
+  case k:
+  of skProc: result.add(defaultParamSeparator)
+  of skMacro: result.add(".m" & defaultParamSeparator)
+  of skMethod: result.add(".e" & defaultParamSeparator)
+  of skIterator: result.add(".i" & defaultParamSeparator)
+  of skTemplate: result.add(".t" & defaultParamSeparator)
+  of skConverter: result.add(".c" & defaultParamSeparator)
+  else: discard
+
+  if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
+    result.add(renderParamTypes(n[paramsPos]))
+
+
+proc isCallable(n: PNode): bool =
+  ## Returns true if `n` contains a callable node.
+  case n.kind
+  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
+    nkConverterDef: result = true
+  else:
+    result = false
+
+
+proc docstringSummary(rstText: string): string =
+  ## Returns just the first line or a brief chunk of text from a rst string.
+  ##
+  ## Most docstrings will contain a one liner summary, so stripping at the
+  ## first newline is usually fine. If after that the content is still too big,
+  ## it is stripped at the first comma, colon or dot, usual english sentence
+  ## separators.
+  ##
+  ## No guarantees are made on the size of the output, but it should be small.
+  ## Also, we hope to not break the rst, but maybe we do. If there is any
+  ## trimming done, an ellipsis unicode char is added.
+  const maxDocstringChars = 100
+  assert (rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#'))
+  result = rstText.substr(2).strip
+  var pos = result.find('\L')
+  if pos > 0:
+    result.delete(pos, result.len - 1)
+    result.add("…")
+  if pos < maxDocstringChars:
+    return
+  # Try to keep trimming at other natural boundaries.
+  pos = result.find({'.', ',', ':'})
+  let last = result.len - 1
+  if pos > 0 and pos < last:
+    result.delete(pos, last)
+    result.add("…")
+
+
 proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   if not isVisible(nameNode): return
-  var name = toRope(getName(d, nameNode))
+  let
+    name = getName(d, nameNode)
+    nameRope = name.toRope
+    plainDocstring = getPlainDocstring(n) # call here before genRecComment!
   var result: PRope = nil
-  var literal = ""
+  var literal, plainName = ""
   var kind = tkEof
   var comm = genRecComment(d, n)  # call this here for the side-effect!
   var r: TSrcGen
+  # Obtain the plain rendered string for hyperlink titles.
+  initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
+    renderNoPragmas, renderNoProcDefs})
+  while true:
+    getNextTok(r, kind, literal)
+    if kind == tkEof:
+      break
+    plainName.add(literal)
+
+  # Render the HTML hyperlink.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
   while true:
     getNextTok(r, kind, literal)
@@ -253,13 +369,47 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
             [toRope(esc(d.target, literal))])
   inc(d.id)
+  let
+    plainNameRope = toRope(xmltree.escape(plainName.strip))
+    cleanPlainSymbol = renderPlainSymbolName(nameNode)
+    complexSymbol = complexName(k, n, cleanPlainSymbol)
+    plainSymbolRope = toRope(cleanPlainSymbol)
+    plainSymbolEncRope = toRope(URLEncode(cleanPlainSymbol))
+    itemIDRope = toRope(d.id)
+    symbolOrId = d.newUniquePlainSymbol(complexSymbol)
+    symbolOrIdRope = symbolOrId.toRope
+    symbolOrIdEncRope = URLEncode(symbolOrId).toRope
+
+  var seeSrcRope: PRope = nil
+  let docItemSeeSrc = getConfigVar("doc.item.seesrc")
+  if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0:
+    let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl,
+      ["path", "line"], [n.info.toFilename.toRope, toRope($n.info.line)])
+    dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc,
+        ["path", "line", "url"], [n.info.toFilename.toRope,
+        toRope($n.info.line), urlRope])])
+
   app(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"),
-                                        ["name", "header", "desc", "itemID"],
+    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-                                        [name, result, comm, toRope(d.id)]))
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"],
+    [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
+      symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope]))
   app(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"),
-                                    ["name", "header", "desc", "itemID"], [
+    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-      toRope(getName(d, nameNode, d.splitAfter)), result, comm, toRope(d.id)]))
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc"],
-  setIndexTerm(d[], $d.id, getName(d, nameNode))
+    [toRope(getName(d, nameNode, d.splitAfter)), result, comm,
+      itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope,
+      plainSymbolEncRope, symbolOrIdEncRope]))
+
+  # Ironically for types the complexSymbol is *cleaner* than the plainName
+  # because it doesn't include object fields or documentation comments. So we
+  # use the plain one for callable elements, and the complex for the rest.
+  var linkTitle = changeFileExt(extractFilename(d.filename), "") & " : "
+  if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip))
+  else: linkTitle.add(xmltree.escape(complexSymbol.strip))
+
+  setIndexTerm(d[], symbolOrId, name, linkTitle,
+    xmltree.escape(plainDocstring.docstringSummary))
 
 proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): PJsonNode =
   if not isVisible(nameNode): return

compiler/options.nim

@@ -100,6 +100,8 @@ var
   gSelectedGC* = gcRefc       # the selected GC
   searchPaths*, lazyPaths*: TLinkedList
   outFile*: string = ""
+  docSeeSrcUrl*: string = ""  # if empty, no seeSrc will be generated. \
+  # The string uses the formatting variables `path` and `line`.
   headerFile*: string = ""
   gVerbosity* = 1             # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors

compiler/renderer.nim

@@ -15,7 +15,7 @@ import
 type 
   TRenderFlag* = enum 
     renderNone, renderNoBody, renderNoComments, renderDocComments, 
-    renderNoPragmas, renderIds
+    renderNoPragmas, renderIds, renderNoProcDefs
   TRenderFlags* = set[TRenderFlag]
   TRenderTok*{.final.} = object 
     kind*: TTokType
@@ -51,10 +51,17 @@ proc isKeyword*(s: string): bool =
       (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): 
     result = true
 
-proc renderDefinitionName*(s: PSym): string =
+proc renderDefinitionName*(s: PSym, noQuotes = false): string =
+  ## Returns the definition name of the symbol.
+  ##
+  ## If noQuotes is false the symbol may be returned in backticks. This will
+  ## happen if the name happens to be a keyword or the first character is not
+  ## part of the SymStartChars set.
   let x = s.name.s
-  if x[0] in SymStartChars and not renderer.isKeyword(x): result = x
+  if noQuotes or (x[0] in SymStartChars and not renderer.isKeyword(x)):
-  else: result = '`' & x & '`'
+    result = x
+  else:
+    result = '`' & x & '`'
 
 const 
   IndentWidth = 2
@@ -1119,22 +1126,22 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkStaticStmt: gstaticStmt(g, n)
   of nkAsmStmt: gasm(g, n)
   of nkProcDef: 
-    putWithSpace(g, tkProc, "proc")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkProc, "proc")
     gproc(g, n)
   of nkConverterDef:
-    putWithSpace(g, tkConverter, "converter")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkConverter, "converter")
     gproc(g, n)
   of nkMethodDef: 
-    putWithSpace(g, tkMethod, "method")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkMethod, "method")
     gproc(g, n)
   of nkIteratorDef: 
-    putWithSpace(g, tkIterator, "iterator")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkIterator, "iterator")
     gproc(g, n)
   of nkMacroDef: 
-    putWithSpace(g, tkMacro, "macro")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkMacro, "macro")
     gproc(g, n)
   of nkTemplateDef: 
-    putWithSpace(g, tkTemplate, "template")
+    if renderNoProcDefs notin g.flags: putWithSpace(g, tkTemplate, "template")
     gproc(g, n)
   of nkTypeSection: 
     gsection(g, n, emptyContext, tkType, "type")
@@ -1336,4 +1343,3 @@ proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) =
     inc(r.idx)
   else: 
     kind = tkEof
-  

compiler/typesrenderer.nim

@@ -0,0 +1,113 @@
+import renderer, strutils, ast, msgs, types
+
+const defaultParamSeparator* = ","
+
+proc renderPlainSymbolName*(n: PNode): string =
+  ## Returns the first non '*' nkIdent node from the tree.
+  ##
+  ## Use this on documentation name nodes to extract the *raw* symbol name,
+  ## without decorations, parameters, or anything. That can be used as the base
+  ## for the HTML hyperlinks.
+  result = ""
+  case n.kind
+  of nkPostfix:
+    for i in 0 .. <n.len:
+      result = renderPlainSymbolName(n[<n.len])
+      if result.len > 0:
+        return
+  of nkIdent:
+    if n.ident.s != "*":
+      result = n.ident.s
+  of nkSym:
+    result = n.sym.renderDefinitionName(noQuotes = true)
+  of nkPragmaExpr:
+    result = renderPlainSymbolName(n[0])
+  of nkAccQuoted:
+    result = renderPlainSymbolName(n[<n.len])
+  else:
+    internalError(n.info, "renderPlainSymbolName() with " & $n.kind)
+  assert (not result.isNil)
+
+proc renderType(n: PNode): string =
+  ## Returns a string with the node type or the empty string.
+  case n.kind:
+  of nkIdent: result = n.ident.s
+  of nkSym: result = typeToString(n.sym.typ)
+  of nkVarTy:
+    assert len(n) == 1
+    result = renderType(n[0])
+  of nkRefTy:
+    assert len(n) == 1
+    result = "ref." & renderType(n[0])
+  of nkPtrTy:
+    assert len(n) == 1
+    result = "ptr." & renderType(n[0])
+  of nkProcTy:
+    assert len(n) > 1
+    let params = n[0]
+    assert params.kind == nkFormalParams
+    assert len(params) > 0
+    result = "proc("
+    for i in 1 .. <len(params): result.add(renderType(params[i]) & ',')
+    result[<len(result)] = ')'
+  of nkIdentDefs:
+    assert len(n) >= 3
+    let typePos = len(n) - 2
+    let typeStr = renderType(n[typePos])
+    result = typeStr
+    for i in 1 .. <typePos:
+      assert n[i].kind == nkIdent
+      result.add(',' & typeStr)
+  of nkTupleTy:
+    assert len(n) > 0
+    result = "tuple["
+    for i in 0 .. <len(n): result.add(renderType(n[i]) & ',')
+    result[<len(result)] = ']'
+  of nkBracketExpr:
+    assert len(n) >= 2
+    result = renderType(n[0]) & '['
+    for i in 1 .. <len(n): result.add(renderType(n[i]) & ',')
+    result[<len(result)] = ']'
+  else: result = ""
+  assert (not result.isNil)
+
+
+proc renderParamTypes(found: var seq[string], n: PNode) =
+  ## Recursive helper, adds to `found` any types, or keeps diving the AST.
+  ##
+  ## The normal `doc` generator doesn't include .typ information, so the
+  ## function won't render types for parameters with default values. The `doc2`
+  ## generator does include the information.
+  case n.kind
+  of nkFormalParams:
+    for i in 1 .. <len(n): renderParamTypes(found, n[i])
+  of nkIdentDefs:
+    # These are parameter names + type + default value node.
+    let typePos = len(n) - 2
+    assert typePos > 0
+    var typeStr = renderType(n[typePos])
+    if typeStr.len < 1:
+      # Try with the last node, maybe its a default value.
+      assert n[typePos+1].kind != nkEmpty
+      let typ = n[typePos+1].typ
+      if not typ.isNil: typeStr = typeToString(typ, preferExported)
+      if typeStr.len < 1:
+        return
+    for i in 0 .. <typePos:
+      assert n[i].kind == nkIdent
+      found.add(typeStr)
+  else:
+    internalError(n.info, "renderParamTypes(found,n) with " & $n.kind)
+
+proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string =
+  ## Returns the types contained in `n` joined by `sep`.
+  ##
+  ## This proc expects to be passed as `n` the parameters of any callable. The
+  ## string output is meant for the HTML renderer. If there are no parameters,
+  ## the empty string is returned. The parameters will be joined by `sep` but
+  ## other characters may appear too, like ``[]`` or ``|``.
+  result = ""
+  var found: seq[string] = @[]
+  renderParamTypes(found, n)
+  if found.len > 0:
+    result = found.join(sep)

config/nimdoc.cfg

@@ -1,6 +1,7 @@
 # This is the config file for the documentation generator.
 # (c) 2012 Andreas Rumpf
-# Feel free to edit the templates as you need.
+# Feel free to edit the templates as you need. If you modify this file, it
+# might be worth updating the hardcoded values in packages/docutils/rstgen.ni
 
 split.item.toc = "20"  
 # too long entries in the table of contents wrap around
@@ -23,17 +24,45 @@ doc.section.toc = """
 </li>
 """
 
+# Chunk of HTML emmited for each entry in the HTML table of contents.
+# Available variables are:
+# * $desc: the actual docstring of the item.
+# * $header: the full version of name, including types, pragmas, tags, etc.
+# * $header_plain: like header but without HTML, for attribute embedding.
+# * $itemID: numerical unique entry of the item in the HTML.
+# * $itemSym: short symbolic name of the item for easier hyperlinking.
+# * $itemSymEnc: quoted version for URLs or attributes.
+# * $itemSymOrID: the symbolic name or the ID if that is not unique.
+# * $itemSymOrIDEnc: quoted version for URLs or attributes.
+# * $name: reduced name of the item.
+# * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
+
 doc.item = """
-<dt id="$itemID"><pre>$header</pre></dt>
+<dt id="$itemSym"><a name="$itemSymOrID"></a><pre>$header</pre></dt>
 <dd>
 $desc
+$seeSrc
 </dd>
 """
 
+# Chunk of HTML emmited for each entry in the HTML table of contents.
+# See doc.item for available substitution variables.
 doc.item.toc = """
-  <li><a class="reference" href="#$itemID">$name</a></li>
+  <li><a class="reference" href="#$itemSymOrID"
+    title="$header_plain">$name</a></li>
 """
 
+# HTML rendered for doc.item's seeSrc variable. Note that this will render to
+# the empty string if you don't pass anything through --docSeeSrcURL. Available
+# substitutaion variables here are:
+# * $path: relative path to the file being processed.
+# * $line: line of the item in the original source file.
+# * $url: whatever you did pass through the --docSeeSrcUrl switch (which also
+#   gets variables path/line replaced!)
+doc.item.seesrc = """<a
+href="https://github.com/Araq/Nimrod/blob/${url}/${path}#L${line}"
+>See source</a>"""
+
 doc.toc = """
 <div class="navigation" id="navigation">
 <ul class="simple">

doc/advopt.txt

@@ -57,6 +57,8 @@ Advanced options:
   --genMapping              generate a mapping file containing
                             (Nimrod, mangled) identifier pairs
   --project                 document the whole project (doc2)
+  --docSeeSrcUrl:url        activate 'see source' for doc and doc2 commands
+                            (see doc.item.seesrc in config/nimdoc.cfg)
   --lineDir:on|off          generation of #line directive on|off
   --embedsrc                embeds the original source code as comments
                             in the generated output

doc/docgen.txt

@@ -146,21 +146,67 @@ Output::
 Related Options
 ===============
 
-``--project`` switch
+Project switch
+--------------
+
 ::
-  nimrod doc2 --project sample
+  nimrod doc2 --project filename.nim
 
 This will recursively generate documentation of all nimrod modules imported
 into the input module, including system modules. Be careful with this command,
 as it may end up sprinkling html files all over your filesystem!
 
 
-``--index`` switch
+Index switch
+------------
+
 ::
-  nimrod doc2 --index:on sample
+  nimrod doc2 --index:on filename.nim
 
 This will generate an index of all the exported symbols in the input Nimrod
-module, and put it into a neighboring file with the extension of `.idx`.
+module, and put it into a neighboring file with the extension of `.idx`. The
+index file is line oriented (newlines have to be escaped). Each line represents
+a tab separated record of several columns, the first two mandatory, the rest
+optional:
+
+1. Mandatory term being indexed. Terms can include quoting according to
+   Nimrod's rules (eg. ```^```)
+2. Base filename plus anchor hyper link (eg.
+   ``algorithm.html#*,int,TSortOrder``).
+3. Optional human readable string to display as hyper link. If the value is not
+   present or is the empty string, the hyper link will be rendered using the
+   term.
+4. Optional title or description of the hyper link. Browsers usually display
+   this as a tooltip after hovering a moment over the hyper link.
+
+Once index files have been generated for one or more modules, the Nimrod
+compiler command ``buildIndex directory`` can be run to go over all the index
+files in the specified directory to generate a `theindex.html <theindex.html>`_
+file.
+
+See source switch
+-----------------
+
+::
+  nimrod doc2 --docSeeSrcUrl:txt filename.nim
+
+When you pass the ``docSeeSrcUrl`` switch to docgen, after each documented item
+in your source code the hyper link *See source* will appear pointing to the
+implementation of that item on a GitHub repository. You can click the link to
+see the implementation of the item.
+
+If you want to reuse this feature in your own documentation you will have to
+modify ``config/nimdoc.cfg`` to contain a ``doc.item.seesrc`` value with a
+hyper link to your own code repository. As you will see by the comments in that
+file, the value ``txt`` passed on the command line will be used in the HTML
+template along others like ``$path`` and ``$line``.
+
+In the case of Nimrod's own documentation, the ``txt`` value is just a commit
+hash to append to a formatted URL to https://github.com/Araq/Nimrod. The
+``tools/nimweb.nim`` helper queries the current git commit hash during doc
+generation, but since you might be working on an unpublished repository, it
+also allows specifying a ``githash`` value in ``web/nimrod.ini`` to force a
+specific commit in the output.
 
 
 Other Input Formats
@@ -183,10 +229,83 @@ command is invoked identically to ``rst2html``, but outputs a .tex file instead
 of .html.
 
 
-Additional Resources
+HTML anchor generation
-=========
+======================
+
+When you run the ``rst2html`` command, all sections in the RST document will
+get an anchor you can hyper link to. Usually you can guess the anchor lower
+casing the section title and replacing spaces with dashes, and in any case you
+can get it from the table of contents. But when you run the ``doc`` or ``doc2``
+commands to generate API documentation, some symbol get one or two anchors at
+the same time: a numerical identifier, or a plain name plus a complex name.
+
+The numerical identifier is just a random number. The number gets assigned
+according to the section and position of the symbol in the file being processed
+and you should not rely on it being constant: if you add or remove a symbol the
+numbers may shuffle around.
+
+The plain name of a symbol is a simplified version of its fully exported
+signature. Variables or constants have the same plain name symbol as their
+complex name. The plain name for procs, templates, and other callable types
+will be their unquoted value after removing parameters, return types and
+pragmas. The plain name allows short and nice linking of symbols which works
+unless you have a module with collisions due to overloading.
+
+If you hyper link a plain name symbol and there are other matches on the same
+HTML file, most browsers will go to the first one. To differentiate the rest,
+you will need to use the complex name. A complex name for a callable type is
+made up from several parts:
+
+    (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\*
+
+The first thing to note is that all callable types have at least a comma, even
+if they don't have any parameters. If there are parameters, they are
+represented by their types and will be comma separated. To the plain symbol a
+suffix may be added depending on the type of the callable:
+
+-------------   --------------
+Callable type   Suffix
+-------------   --------------
+proc            *empty string*
+macro           ``.m``
+method          ``.e``
+iterator        ``.i``
+template        ``.t``
+converter       ``.c``
+-------------   --------------
+
+The relationship of type to suffix is made by the proc ``complexName`` in the
+``compiler/docgen.nim`` file. Here are some examples of complex names for
+symbols in the `system module <system.html>`_.
+
+* ``type TSignedInt = int | int8 | int16 | int32 | int64`` **=>**
+  `#TSignedInt <system.html#TSignedInt>`_
+* ``var globalRaiseHook: proc (e: ref E_Base): bool {.nimcall.}`` **=>**
+  `#globalRaiseHook <system.html#globalRaiseHook>`_
+* ``const NimrodVersion = "0.0.0"`` **=>**
+  `#NimrodVersion <system.html#NimrodVersion>`_
+* ``proc getTotalMem(): int {.rtl, raises: [], tags: [].}`` **=>**
+  `#getTotalMem, <system.html#getTotalMem,>`_
+* ``proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}`` **=>**
+  `#len,seq[T] <system.html#len,seq[T]>`_
+* ``iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}`` **=>**
+  `#pairs.i,seq[T] <system.html#pairs.i,seq[T]>`_
+* ``template newException[](exceptn: typedesc; message: string): expr`` **=>**
+  `#newException.t,typedesc,string
+  <system.html#newException.t,typedesc,string>`_
+
+
+Additional resources
+====================
 
 `Nimrod Compiler User Guide <nimrodc.html#command-line-switches>`_
 
 `RST Quick Reference
 <http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_
+
+The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and
+``config/nimdoc.tex.cfg`` configuration files. You can add and modify these
+files to your project to change the look of docgen output.
+
+You can import the `packages/docutils/rstgen module <rstgen.html>`_ in your
+programs if you want to reuse the compiler's documentation generation procs.

lib/core/macros.nim

@@ -598,7 +598,7 @@ template badnodekind(k; f): stmt{.immediate.} =
 
 proc body*(someProc: PNimrodNode): PNimrodNode {.compileTime.} =
   case someProc.kind:
-  of routineNodes:
+  of RoutineNodes:
     return someProc[6]
   of nnkBlockStmt, nnkWhileStmt:
     return someProc[1]
@@ -609,7 +609,7 @@ proc body*(someProc: PNimrodNode): PNimrodNode {.compileTime.} =
 
 proc `body=`*(someProc: PNimrodNode, val: PNimrodNode) {.compileTime.} =
   case someProc.kind 
-  of routineNodes:
+  of RoutineNodes:
     someProc[6] = val
   of nnkBlockStmt, nnkWhileStmt:
     someProc[1] = val

lib/packages/docutils/rstgen.nim

@@ -9,11 +9,14 @@
 
 ## This module implements a generator of HTML/Latex from
 ## `reStructuredText`:idx: (see http://docutils.sourceforge.net/rst.html for
-## information on this markup syntax). You can generate HTML output through the
+## information on this markup syntax) and is used by the compiler's `docgen
-## convenience proc ``rstToHtml``, which provided an input string with rst
+## tools <docgen.html>`_.
-## markup returns a string with the generated HTML. The final output is meant
+##
-## to be embedded inside a full document you provide yourself, so it won't
+## You can generate HTML output through the convenience proc ``rstToHtml``,
-## contain the usual ``<header>`` or ``<body>`` parts.
+## which provided an input string with rst markup returns a string with the
+## generated HTML. The final output is meant to be embedded inside a full
+## document you provide yourself, so it won't contain the usual ``<header>`` or
+## ``<body>`` parts.
 ##
 ## You can also create a ``TRstGenerator`` structure and populate it with the
 ## other lower level methods to finally build complete documents. This requires
@@ -50,6 +53,9 @@ type
     msgHandler*: TMsgHandler
     filename*: string
     meta*: array[TMetaEnum, string]
+    currentSection: string ## \
+    ## Stores the empty string or the last headline/overline found in the rst
+    ## document, so it can be used as a prettier name for term index generation.
   
   PDoc = var TRstGenerator ## Alias to type less.
 
@@ -104,6 +110,7 @@ proc initRstGenerator*(g: var TRstGenerator, target: TOutputTarget,
   g.theIndex = ""
   g.options = options
   g.findFile = findFile
+  g.currentSection = ""
   g.msgHandler = msgHandler
   
   let s = config["split.item.toc"]
@@ -227,20 +234,44 @@ proc renderAux(d: PDoc, n: PRstNode, frmtA, frmtB: string, result: var string) =
 
 # ---------------- index handling --------------------------------------------
 
-proc setIndexTerm*(d: var TRstGenerator, id, term: string) =
+proc quoteIndexColumn(text: string): string =
+  ## Returns a safe version of `text` for serialization to the ``.idx`` file.
+  ##
+  ## The returned version can be put without worries in a line based tab
+  ## separated column text file. The following character sequence replacements
+  ## will be performed for that goal:
+  ##
+  ## * ``"\\"`` => ``"\\\\"``
+  ## * ``"\n"`` => ``"\\n"``
+  ## * ``"\t"`` => ``"\\t"``
+  result = text.replace("\\", "\\\\").replace("\n", "\\n").replace("\t", "\\t")
+
+proc unquoteIndexColumn(text: string): string =
+  ## Returns the unquoted version generated by ``quoteIndexColumn``.
+  result = text.replace("\\t", "\t").replace("\\n", "\n").replace("\\\\", "\\")
+
+proc setIndexTerm*(d: var TRstGenerator, id, term: string,
+                   linkTitle, linkDesc = "") =
   ## Adds a `term` to the index using the specified hyperlink identifier.
   ##
   ## The ``d.theIndex`` string will be used to append the term in the format
   ## ``term<tab>file#id``. The anchor will be the based on the name of the file
   ## currently being parsed plus the `id`, which will be appended after a hash.
+  ## If `linkTitle` or `linkDesc` are not the empty string, two additional
+  ## columns with their contents will be added.
   ##
-  ## The index won't be written to disk unless you call ``writeIndexFile``.
+  ## The index won't be written to disk unless you call ``writeIndexFile``. The
+  ## purpose of the index is documented in the `docgen tools guide
+  ## <docgen.html#index-switch>`_.
   d.theIndex.add(term)
   d.theIndex.add('\t')
   let htmlFile = changeFileExt(extractFilename(d.filename), HtmlExt)
   d.theIndex.add(htmlFile)
   d.theIndex.add('#')
   d.theIndex.add(id)
+  if linkTitle.len > 0 or linkDesc.len > 0:
+    d.theIndex.add('\t' & linkTitle.quoteIndexColumn)
+    d.theIndex.add('\t' & linkDesc.quoteIndexColumn)
   d.theIndex.add("\n")
 
 proc hash(n: PRstNode): int =
@@ -256,7 +287,7 @@ proc renderIndexTerm(d: PDoc, n: PRstNode, result: var string) =
   let id = rstnodeToRefname(n) & '_' & $abs(hash(n))
   var term = ""
   renderAux(d, n, term)
-  setIndexTerm(d, id, term)
+  setIndexTerm(d, id, term, d.currentSection)
   dispA(d.target, result, "<span id=\"$1\">$2</span>", "$2\\label{$1}", 
         [id, term])
 
@@ -264,13 +295,22 @@ type
   TIndexEntry {.pure, final.} = object
     keyword: string
     link: string
+    linkTitle: string ## If not nil, contains a prettier text for the href
+    linkDesc: string ## If not nil, the title attribute of the final href
 
 proc cmp(a, b: TIndexEntry): int =
+  ## Sorts two ``TIndexEntry`` first by `keyword` field, then by `link`.
   result = cmpIgnoreStyle(a.keyword, b.keyword)
+  if result == 0:
+    result = cmpIgnoreStyle(a.link, b.link)
 
 proc `<-`(a: var TIndexEntry, b: TIndexEntry) =
   shallowCopy a.keyword, b.keyword
   shallowCopy a.link, b.link
+  if b.linkTitle.isNil: a.linkTitle = nil
+  else: shallowCopy a.linkTitle, b.linkTitle
+  if b.linkDesc.isNil: a.linkDesc = nil
+  else: shallowCopy a.linkDesc, b.linkDesc
 
 proc sortIndex(a: var openArray[TIndexEntry]) =
   # we use shellsort here; fast and simple
@@ -307,6 +347,15 @@ proc mergeIndexes*(dir: string): string =
         setLen(a, L+1)
         a[L].keyword = line.substr(0, s-1)
         a[L].link = line.substr(s+1)
+        if a[L].link.find('\t') > 0:
+          let extraCols = a[L].link.split('\t')
+          a[L].link = extraCols[0]
+          assert extraCols.len == 3
+          a[L].linkTitle = extraCols[1].unquoteIndexColumn
+          a[L].linkDesc = extraCols[2].unquoteIndexColumn
+        else:
+          a[L].linkTitle = nil
+          a[L].linkDesc = nil
         inc L
   sortIndex(a)
   result = ""
@@ -316,9 +365,17 @@ proc mergeIndexes*(dir: string): string =
                 [a[i].keyword])
     var j = i
     while j < L and a[i].keyword == a[j].keyword:
-      result.addf(
+      let
-        "<li><a class=\"reference external\" href=\"$1\">$1</a></li>\n", 
+        url = a[j].link
-        [a[j].link])
+        text = if not a[j].linkTitle.isNil: a[j].linkTitle else: url
+        desc = if not a[j].linkDesc.isNil: a[j].linkDesc else: ""
+      if desc.len > 0:
+        result.addf("""<li><a class="reference external"
+          title="$3" href="$1">$2</a></li>
+          """, [url, text, desc])
+      else:
+        result.addf("""<li><a class="reference external" href="$1">$2</a></li>
+          """, [url, text])
       inc j
     result.add("</ul></dd>\n")
     i = j
@@ -328,6 +385,7 @@ proc mergeIndexes*(dir: string): string =
 proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = 
   var tmp = ""
   for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
+  d.currentSection = tmp
   var refname = rstnodeToRefname(n)
   if d.hasToc:
     var length = len(d.tocPart)
@@ -349,14 +407,17 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
   
 proc renderOverline(d: PDoc, n: PRstNode, result: var string) = 
   if d.meta[metaTitle].len == 0:
+    d.currentSection = d.meta[metaTitle]
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaTitle])
   elif d.meta[metaSubtitle].len == 0:
+    d.currentSection = d.meta[metaSubtitle]
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaSubtitle])
   else:
     var tmp = ""
     for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
+    d.currentSection = tmp
     dispA(d.target, result, "<h$1 id=\"$2\"><center>$3</center></h$1>", 
                    "\\rstov$4{$3}\\label{$2}\n", [$n.level,
         rstnodeToRefname(n), tmp, $chr(n.level - 1 + ord('A'))])
@@ -716,6 +777,8 @@ proc defaultConfig*(): PStringTable =
   template setConfigVar(key, val: expr) =
     result[key] = val
   
+  # If you need to modify these values, it might be worth updating the template
+  # file in config/nimdoc.cfg.
   setConfigVar("split.item.toc", "20")
   setConfigVar("doc.section", """
 <div class="section" id="$sectionID">
@@ -733,13 +796,14 @@ $content
 </li>
 """)
   setConfigVar("doc.item", """
-<dt id="$itemID"><pre>$header</pre></dt>
+<dt id="$itemID"><a name="$itemSymOrIDEnc"></a><pre>$header</pre></dt>
 <dd>
 $desc
 </dd>
 """)
   setConfigVar("doc.item.toc", """
-  <li><a class="reference" href="#$itemID">$name</a></li>
+  <li><a class="reference" href="#$itemSymOrIDEnc"
+    title="$header_plain">$name</a></li>
 """)
   setConfigVar("doc.toc", """
 <div class="navigation" id="navigation">

lib/pure/cgi.nim

@@ -74,7 +74,7 @@ proc URLdecode*(s: string): string =
     inc(j)
   setLen(result, j)
 
-proc addXmlChar(dest: var string, c: Char) {.inline.} =
+proc addXmlChar(dest: var string, c: char) {.inline.} =
   case c
   of '&': add(dest, "&")
   of '<': add(dest, "&lt;")

tools/nimweb.nim

@@ -9,7 +9,7 @@
 
 import
   os, strutils, times, parseopt, parsecfg, streams, strtabs, tables,
-  re, htmlgen, macros, md5
+  re, htmlgen, macros, md5, osproc
 
 type
   TKeyValPair = tuple[key, id, val: string]
@@ -19,6 +19,7 @@ type
     authors, projectName, projectTitle, logo, infile, outdir, ticker: string
     vars: PStringTable
     nimrodArgs: string
+    gitCommit: string
     quotations: TTable[string, tuple[quote, author: string]]
   TRssItem = object
     year, month, day, title: string
@@ -40,6 +41,11 @@ proc initConfigData(c: var TConfigData) =
   c.logo = ""
   c.ticker = ""
   c.vars = newStringTable(modeStyleInsensitive)
+  c.gitCommit = "master"
+  # Attempts to obtain the git current commit.
+  let (output, code) = execCmdEx("git log -n 1 --format=%H")
+  if code == 0 and output.strip.len == 40:
+    c.gitCommit = output.strip
   c.quotations = initTable[string, tuple[quote, author: string]]()
 
 include "website.tmpl"
@@ -197,6 +203,11 @@ proc parseIniFile(c: var TConfigData) =
       c.outdir = splitFile(c.infile).dir
   else:
     quit("cannot open: " & c.infile)
+  # Ugly hack to override git command output when building private repo.
+  if c.vars.hasKey("githash"):
+    let githash = c.vars["githash"].strip
+    if githash.len > 0:
+      c.gitCommit = githash
 
 # ------------------- main ----------------------------------------------------
 
@@ -219,14 +230,17 @@ proc buildDocSamples(c: var TConfigData, destPath: string) =
 proc buildDoc(c: var TConfigData, destPath: string) =
   # call nim for the documentation:
   for d in items(c.doc):
-    exec("nimrod rst2html $# -o:$# --index:on $#" %
+    exec("nimrod rst2html $# --docSeeSrcUrl:$# -o:$# --index:on $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
+      [c.nimrodArgs, c.gitCommit,
+      destPath / changeFileExt(splitFile(d).name, "html"), d])
   for d in items(c.srcdoc):
-    exec("nimrod doc $# -o:$# --index:on $#" %
+    exec("nimrod doc $# --docSeeSrcUrl:$# -o:$# --index:on $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
+      [c.nimrodArgs, c.gitCommit,
+      destPath / changeFileExt(splitFile(d).name, "html"), d])
   for d in items(c.srcdoc2):
-    exec("nimrod doc2 $# -o:$# --index:on $#" %
+    exec("nimrod doc2 $# --docSeeSrcUrl:$# -o:$# --index:on $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
+      [c.nimrodArgs, c.gitCommit,
+      destPath / changeFileExt(splitFile(d).name, "html"), d])
   exec("nimrod buildIndex -o:$1/theindex.html $1" % [destPath])
 
 proc buildPdfDoc(c: var TConfigData, destPath: string) =
@@ -251,8 +265,9 @@ proc buildPdfDoc(c: var TConfigData, destPath: string) =
 proc buildAddDoc(c: var TConfigData, destPath: string) =
   # build additional documentation (without the index):
   for d in items(c.webdoc):
-    exec("nimrod doc $# -o:$# $#" %
+    exec("nimrod doc $# --docSeeSrcUrl:$# -o:$# $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
+      [c.nimrodArgs, c.gitCommit,
+      destPath / changeFileExt(splitFile(d).name, "html"), d])
 
 proc parseNewsTitles(inputFilename: string): seq[TRssItem] =
   # parses the file for titles and returns them as TRssItem blocks.

tools/website.tmpl

@@ -6,7 +6,7 @@
 
 <head>
   <title>$c.projectTitle</title>
-  <meta http-equiv="content-type" content="text/html; charset=iso-8859-1" />
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
   <link rel="stylesheet" type="text/css" href="assets/style.css" />
   #if len(rss) > 0:
   <link href="$rss" title="Recent changes" type="application/atom+xml" rel="alternate">