From 3e25d5f24747c324a5a0f0deb555611166c96209 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Apr 2014 01:19:15 +0200 Subject: [PATCH] merged better html links #850 --- compiler/commands.nim | 3 + compiler/docgen.nim | 168 +++++++++++++++++++++++++++++-- compiler/options.nim | 2 + compiler/renderer.nim | 28 ++++-- compiler/typesrenderer.nim | 113 +++++++++++++++++++++ config/nimdoc.cfg | 35 ++++++- doc/advopt.txt | 2 + doc/docgen.txt | 133 ++++++++++++++++++++++-- lib/core/macros.nim | 4 +- lib/packages/docutils/rstgen.nim | 90 ++++++++++++++--- lib/pure/cgi.nim | 2 +- tools/nimweb.nim | 33 ++++-- tools/website.tmpl | 2 +- 13 files changed, 559 insertions(+), 56 deletions(-) create mode 100644 compiler/typesrenderer.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 18bdb54d3..8339219ed 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -267,6 +267,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "out", "o": expectArg(switch, arg, pass, info) options.outFile = arg + of "docseesrcurl": + expectArg(switch, arg, pass, info) + options.docSeeSrcUrl = arg of "mainmodule", "m": expectArg(switch, arg, pass, info) optMainModule = arg diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d8c439c9c..6948c4979 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -14,7 +14,7 @@ import ast, strutils, strtabs, options, msgs, os, ropes, idents, wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, - importer, sempass2, json + importer, sempass2, json, xmltree, cgi, typesrenderer type TSections = array[TSymKind, PRope] @@ -23,8 +23,9 @@ type id: int # for generating IDs toc, section: TSections indexValFilename: string + seenSymbols: PStringTable # avoids duplicate symbol generation for HTML. - PDoc* = ref TDocumentor + PDoc* = ref TDocumentor ## Alias to type less. proc compilerMsgHandler(filename: string, line, col: int, msgKind: rst.TMsgKind, arg: string) {.procvar.} = @@ -59,6 +60,7 @@ proc newDocumentor*(filename: string, config: PStringTable): PDoc = initRstGenerator(result[], (if gCmd != cmdRst2tex: outHtml else: outLatex), options.gConfigVars, filename, {roSupportRawDirective}, docgenFindFile, compilerMsgHandler) + result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 proc dispA(dest: var PRope, xml, tex: string, args: openArray[PRope]) = @@ -144,6 +146,23 @@ proc genRecComment(d: PDoc, n: PNode): PRope = else: n.comment = nil +proc getPlainDocstring(n: PNode): string = + ## Gets the plain text docstring of a node non destructively. + ## + ## You need to call this before genRecComment, whose side effects are removal + ## of comments from the tree. The proc will recursively scan and return all + ## the concatenated ``##`` comments of the node. + result = "" + if n == nil: return + if n.comment != nil and startsWith(n.comment, "##"): + result = n.comment + if result.len < 1: + if n.kind notin {nkEmpty..nkNilLit}: + for i in countup(0, len(n)-1): + result = getPlainDocstring(n.sons[i]) + if result.len > 0: return + + proc findDocComment(n: PNode): PNode = if n == nil: return nil if not isNil(n.comment) and startsWith(n.comment, "##"): return n @@ -205,14 +224,111 @@ proc getRstName(n: PNode): PRstNode = internalError(n.info, "getRstName()") result = nil +proc newUniquePlainSymbol(d: PDoc, original: string): string = + ## Returns a new unique plain symbol made up from the original. + ## + ## When a collision is found in the seenSymbols table, new numerical variants + ## with underscore + number will be generated. + if not d.seenSymbols.hasKey(original): + result = original + d.seenSymbols[original] = "" + return + + # Iterate over possible numeric variants of the original name. + var count = 2 + + while true: + result = original & "_" & $count + if not d.seenSymbols.hasKey(result): + d.seenSymbols[result] = "" + break + count += 1 + + +proc complexName(k: TSymKind, n: PNode, baseName: string): string = + ## Builds a complex unique href name for the node. + ## + ## Pass as ``baseName`` the plain symbol obtained from the nodeName. The + ## format of the returned symbol will be ``baseName(.callable type)?,(param + ## type)?(,param type)*``. The callable type part will be added only if the + ## node is not a proc, as those are the common ones. The suffix will be a dot + ## and a single letter representing the type of the callable. The parameter + ## types will be added with a preceeding dash. Return types won't be added. + ## + ## If you modify the output of this proc, please update the anchor generation + ## section of ``doc/docgen.txt``. + result = baseName + case k: + of skProc: result.add(defaultParamSeparator) + of skMacro: result.add(".m" & defaultParamSeparator) + of skMethod: result.add(".e" & defaultParamSeparator) + of skIterator: result.add(".i" & defaultParamSeparator) + of skTemplate: result.add(".t" & defaultParamSeparator) + of skConverter: result.add(".c" & defaultParamSeparator) + else: discard + + if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams: + result.add(renderParamTypes(n[paramsPos])) + + +proc isCallable(n: PNode): bool = + ## Returns true if `n` contains a callable node. + case n.kind + of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, + nkConverterDef: result = true + else: + result = false + + +proc docstringSummary(rstText: string): string = + ## Returns just the first line or a brief chunk of text from a rst string. + ## + ## Most docstrings will contain a one liner summary, so stripping at the + ## first newline is usually fine. If after that the content is still too big, + ## it is stripped at the first comma, colon or dot, usual english sentence + ## separators. + ## + ## No guarantees are made on the size of the output, but it should be small. + ## Also, we hope to not break the rst, but maybe we do. If there is any + ## trimming done, an ellipsis unicode char is added. + const maxDocstringChars = 100 + assert (rstText.len < 2 or (rstText[0] == '#' and rstText[1] == '#')) + result = rstText.substr(2).strip + var pos = result.find('\L') + if pos > 0: + result.delete(pos, result.len - 1) + result.add("…") + if pos < maxDocstringChars: + return + # Try to keep trimming at other natural boundaries. + pos = result.find({'.', ',', ':'}) + let last = result.len - 1 + if pos > 0 and pos < last: + result.delete(pos, last) + result.add("…") + + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = if not isVisible(nameNode): return - var name = toRope(getName(d, nameNode)) + let + name = getName(d, nameNode) + nameRope = name.toRope + plainDocstring = getPlainDocstring(n) # call here before genRecComment! var result: PRope = nil - var literal = "" + var literal, plainName = "" var kind = tkEof var comm = genRecComment(d, n) # call this here for the side-effect! var r: TSrcGen + # Obtain the plain rendered string for hyperlink titles. + initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, + renderNoPragmas, renderNoProcDefs}) + while true: + getNextTok(r, kind, literal) + if kind == tkEof: + break + plainName.add(literal) + + # Render the HTML hyperlink. initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) while true: getNextTok(r, kind, literal) @@ -253,13 +369,47 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = dispA(result, "$1", "\\spanOther{$1}", [toRope(esc(d.target, literal))]) inc(d.id) + let + plainNameRope = toRope(xmltree.escape(plainName.strip)) + cleanPlainSymbol = renderPlainSymbolName(nameNode) + complexSymbol = complexName(k, n, cleanPlainSymbol) + plainSymbolRope = toRope(cleanPlainSymbol) + plainSymbolEncRope = toRope(URLEncode(cleanPlainSymbol)) + itemIDRope = toRope(d.id) + symbolOrId = d.newUniquePlainSymbol(complexSymbol) + symbolOrIdRope = symbolOrId.toRope + symbolOrIdEncRope = URLEncode(symbolOrId).toRope + + var seeSrcRope: PRope = nil + let docItemSeeSrc = getConfigVar("doc.item.seesrc") + if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0: + let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl, + ["path", "line"], [n.info.toFilename.toRope, toRope($n.info.line)]) + dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc, + ["path", "line", "url"], [n.info.toFilename.toRope, + toRope($n.info.line), urlRope])]) + app(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"), - ["name", "header", "desc", "itemID"], - [name, result, comm, toRope(d.id)])) + ["name", "header", "desc", "itemID", "header_plain", "itemSym", + "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"], + [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope, + symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope])) app(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"), - ["name", "header", "desc", "itemID"], [ - toRope(getName(d, nameNode, d.splitAfter)), result, comm, toRope(d.id)])) - setIndexTerm(d[], $d.id, getName(d, nameNode)) + ["name", "header", "desc", "itemID", "header_plain", "itemSym", + "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc"], + [toRope(getName(d, nameNode, d.splitAfter)), result, comm, + itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope, + plainSymbolEncRope, symbolOrIdEncRope])) + + # Ironically for types the complexSymbol is *cleaner* than the plainName + # because it doesn't include object fields or documentation comments. So we + # use the plain one for callable elements, and the complex for the rest. + var linkTitle = changeFileExt(extractFilename(d.filename), "") & " : " + if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip)) + else: linkTitle.add(xmltree.escape(complexSymbol.strip)) + + setIndexTerm(d[], symbolOrId, name, linkTitle, + xmltree.escape(plainDocstring.docstringSummary)) proc genJSONItem(d: PDoc, n, nameNode: PNode, k: TSymKind): PJsonNode = if not isVisible(nameNode): return diff --git a/compiler/options.nim b/compiler/options.nim index 102ebc386..fa8b77ead 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -100,6 +100,8 @@ var gSelectedGC* = gcRefc # the selected GC searchPaths*, lazyPaths*: TLinkedList outFile*: string = "" + docSeeSrcUrl*: string = "" # if empty, no seeSrc will be generated. \ + # The string uses the formatting variables `path` and `line`. headerFile*: string = "" gVerbosity* = 1 # how verbose the compiler is gNumberOfProcessors*: int # number of processors diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 2d2310914..fa119eba9 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -15,7 +15,7 @@ import type TRenderFlag* = enum renderNone, renderNoBody, renderNoComments, renderDocComments, - renderNoPragmas, renderIds + renderNoPragmas, renderIds, renderNoProcDefs TRenderFlags* = set[TRenderFlag] TRenderTok*{.final.} = object kind*: TTokType @@ -51,10 +51,17 @@ proc isKeyword*(s: string): bool = (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): result = true -proc renderDefinitionName*(s: PSym): string = +proc renderDefinitionName*(s: PSym, noQuotes = false): string = + ## Returns the definition name of the symbol. + ## + ## If noQuotes is false the symbol may be returned in backticks. This will + ## happen if the name happens to be a keyword or the first character is not + ## part of the SymStartChars set. let x = s.name.s - if x[0] in SymStartChars and not renderer.isKeyword(x): result = x - else: result = '`' & x & '`' + if noQuotes or (x[0] in SymStartChars and not renderer.isKeyword(x)): + result = x + else: + result = '`' & x & '`' const IndentWidth = 2 @@ -1119,22 +1126,22 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = of nkStaticStmt: gstaticStmt(g, n) of nkAsmStmt: gasm(g, n) of nkProcDef: - putWithSpace(g, tkProc, "proc") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkProc, "proc") gproc(g, n) of nkConverterDef: - putWithSpace(g, tkConverter, "converter") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkConverter, "converter") gproc(g, n) of nkMethodDef: - putWithSpace(g, tkMethod, "method") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkMethod, "method") gproc(g, n) of nkIteratorDef: - putWithSpace(g, tkIterator, "iterator") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkIterator, "iterator") gproc(g, n) of nkMacroDef: - putWithSpace(g, tkMacro, "macro") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkMacro, "macro") gproc(g, n) of nkTemplateDef: - putWithSpace(g, tkTemplate, "template") + if renderNoProcDefs notin g.flags: putWithSpace(g, tkTemplate, "template") gproc(g, n) of nkTypeSection: gsection(g, n, emptyContext, tkType, "type") @@ -1336,4 +1343,3 @@ proc getNextTok(r: var TSrcGen, kind: var TTokType, literal: var string) = inc(r.idx) else: kind = tkEof - diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim new file mode 100644 index 000000000..ebb9b7e15 --- /dev/null +++ b/compiler/typesrenderer.nim @@ -0,0 +1,113 @@ +import renderer, strutils, ast, msgs, types + +const defaultParamSeparator* = "," + +proc renderPlainSymbolName*(n: PNode): string = + ## Returns the first non '*' nkIdent node from the tree. + ## + ## Use this on documentation name nodes to extract the *raw* symbol name, + ## without decorations, parameters, or anything. That can be used as the base + ## for the HTML hyperlinks. + result = "" + case n.kind + of nkPostfix: + for i in 0 .. 0: + return + of nkIdent: + if n.ident.s != "*": + result = n.ident.s + of nkSym: + result = n.sym.renderDefinitionName(noQuotes = true) + of nkPragmaExpr: + result = renderPlainSymbolName(n[0]) + of nkAccQuoted: + result = renderPlainSymbolName(n[ 1 + let params = n[0] + assert params.kind == nkFormalParams + assert len(params) > 0 + result = "proc(" + for i in 1 .. = 3 + let typePos = len(n) - 2 + let typeStr = renderType(n[typePos]) + result = typeStr + for i in 1 .. 0 + result = "tuple[" + for i in 0 .. = 2 + result = renderType(n[0]) & '[' + for i in 1 .. 0 + var typeStr = renderType(n[typePos]) + if typeStr.len < 1: + # Try with the last node, maybe its a default value. + assert n[typePos+1].kind != nkEmpty + let typ = n[typePos+1].typ + if not typ.isNil: typeStr = typeToString(typ, preferExported) + if typeStr.len < 1: + return + for i in 0 .. 0: + result = found.join(sep) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 63a3c30c4..dc241db50 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -1,6 +1,7 @@ # This is the config file for the documentation generator. # (c) 2012 Andreas Rumpf -# Feel free to edit the templates as you need. +# Feel free to edit the templates as you need. If you modify this file, it +# might be worth updating the hardcoded values in packages/docutils/rstgen.ni split.item.toc = "20" # too long entries in the table of contents wrap around @@ -23,17 +24,45 @@ doc.section.toc = """ """ +# Chunk of HTML emmited for each entry in the HTML table of contents. +# Available variables are: +# * $desc: the actual docstring of the item. +# * $header: the full version of name, including types, pragmas, tags, etc. +# * $header_plain: like header but without HTML, for attribute embedding. +# * $itemID: numerical unique entry of the item in the HTML. +# * $itemSym: short symbolic name of the item for easier hyperlinking. +# * $itemSymEnc: quoted version for URLs or attributes. +# * $itemSymOrID: the symbolic name or the ID if that is not unique. +# * $itemSymOrIDEnc: quoted version for URLs or attributes. +# * $name: reduced name of the item. +# * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used). + doc.item = """ -
$header
+
$header
$desc +$seeSrc
""" +# Chunk of HTML emmited for each entry in the HTML table of contents. +# See doc.item for available substitution variables. doc.item.toc = """ -
  • $name
    • +
  • $name
    • """ +# HTML rendered for doc.item's seeSrc variable. Note that this will render to +# the empty string if you don't pass anything through --docSeeSrcURL. Available +# substitutaion variables here are: +# * $path: relative path to the file being processed. +# * $line: line of the item in the original source file. +# * $url: whatever you did pass through the --docSeeSrcUrl switch (which also +# gets variables path/line replaced!) +doc.item.seesrc = """See source""" + doc.toc = """