jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

docgen: implement cross-document links (#20990)

Browse source 
* docgen: implement cross-document links

Fully implements https://github.com/nim-lang/RFCs/issues/125
Follow-up of: https://github.com/nim-lang/Nim/pull/18642 (for internal links)
and https://github.com/nim-lang/Nim/issues/20127.

Overview
--------

Explicit import-like directive is required, called `.. importdoc::`.
(the syntax is % RST, Markdown will use it for a while).

Then one can reference any symbols/headings/anchors, as if they
were in the local file (but they will be prefixed with a module name
or markup document in link text).
It's possible to reference anything from anywhere (any direction
in `.nim`/`.md`/`.rst` files).

See `doc/docgen.md` for full description.

Working is based on `.idx` files, hence one needs to generate
all `.idx` beforehand. A dedicated option `--index:only` is introduced
(and a separate stage for `--index:only` is added to `kochdocs.nim`).

Performance note
----------------

Full run for `./koch docs` now takes 185% of the time before this PR.
(After: 315 s, before: 170 s on my PC).
All the time seems to be spent on `--index:only` run, which takes
almost as much (85%) of normal doc run -- it seems that most time
is spent on file parsing, turning off HTML generation phase has not
helped much.
(One could avoid it by specifying list of files that can be referenced
and pre-processing only them. But it can become error-prone and I assume
that these linke will be **everywhere** in the repository anyway,
especially considering https://github.com/nim-lang/RFCs/issues/478.
So every `.nim`/`.md` file is processed for `.idx` first).

But that's all without significant part of repository converted to
cross-module auto links. To estimate impact I checked the time for
`doc`ing a few files (after all indexes have been generated), and
everywhere difference was **negligible**.
E.g. for `lib/std/private/osfiles.nim` that `importdoc`s large
`os.idx` and hence should have been a case with relatively large
performance impact, but:

* After: 0.59 s.
* Before: 0.59 s.

So Nim compiler works so slow that doc part basically does not matter :-)

Testing
-------

1) added `extlinks` test to `nimdoc/`
2) checked that `theindex.html` is still correct
2) fixed broken auto-links for modules that were derived from `os.nim`
   by adding appropriate ``importdoc``

Implementation note
-------------------

Parsing and formating of `.idx` entries is moved into a dedicated
`rstidx.nim` module from `rstgen.nim`.

`.idx` file format changed:

* fields are not escaped in most cases because we need original
  strings for referencing, not HTML ones
  (the exception is linkTitle for titles and headings).
  Escaping happens later -- on the stage of `rstgen` buildIndex, etc.
* all lines have fixed number of columns 6
* added discriminator tag as a first column,
  it always allows distinguish Nim/markup entries, titles/headings, etc.
  `rstgen` does not rely any more (in most cases) on ad-hoc logic
  to determine what type each entry is.
* there is now always a title entry added at the first line.
* add a line number as 6th column
* linkTitle (4th) column has a different format: before it was like
  `module: funcName()`, now it's `proc funcName()`.
  (This format is also propagated to `theindex.html` and search results,
  I kept it that way since I like it more though it's discussible.)
  This column is what used for Nim symbols resolution.
* also changed details on column format for headings and titles:
  "keyword" is original, "linkTitle" is HTML one

* fix paths on Windows + more clear code

* Update compiler/docgen.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Handle .md and .nim paths uniformly in findRefFile

* handle titles better + more comments

* don't allow markup overwrite index title for .nim files

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>
This commit is contained in:
Andrey Makarov 2023-01-04 23:19:01 +03:00 • committed by GitHub
commit 2620da9bf9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 1863 additions and 491 deletions
Download patch file Download diff file Expand all files Collapse all files

200
compiler/docgen.nim
View file

@ -7,13 +7,17 @@
# distribution, for details about the copyright.
#
# This is the documentation generator. Cross-references are generated
# by knowing how the anchors are going to be named.
## This is the Nim documentation generator. Cross-references are generated
## by knowing how the anchors are going to be named.
##
## .. importdoc:: ../docgen.md
##
## For corresponding users' documentation see [Nim DocGen Tools Guide].
import
ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
wordrecg, syntaxes, renderer, lexer,
packages/docutils/[rst, rstgen, dochelpers],
packages/docutils/[rst, rstidx, rstgen, dochelpers],
json, xmltree, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, tables, nimpaths, renderverbatim, osproc, packages
@ -91,7 +95,7 @@ type
jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering
types: TStrTable
sharedState: PRstSharedState
standaloneDoc: bool
standaloneDoc: bool # is markup (.rst/.md) document?
conf*: ConfigRef
cache*: IdentCache
exampleCounter: int
@ -225,7 +229,7 @@ proc attachToType(d: PDoc; p: PSym): PSym =
if params.len > 0: check(0)
for i in 2..<params.len: check(i)
template declareClosures =
template declareClosures(currentFilename: AbsoluteFile, destFile: string) =
proc compilerMsgHandler(filename: string, line, col: int,
msgKind: rst.MsgKind, arg: string) {.gcsafe.} =
# translate msg kind:
@ -249,6 +253,7 @@ template declareClosures =
of mwBrokenLink: k = warnRstBrokenLink
of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
of mwUnsupportedField: k = warnRstFieldXNotSupported
of mwUnusedImportdoc: k = warnRstUnusedImportdoc
of mwRstStyle: k = warnRstStyle
{.gcsafe.}:
globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
@ -259,10 +264,29 @@ template declareClosures =
result = getCurrentDir() / s
if not fileExists(result): result = ""
proc docgenFindRefFile(targetRelPath: string):
tuple[targetPath: string, linkRelPath: string] {.gcsafe.} =
let fromDir = splitFile(destFile).dir # dir where we reference from
let basedir = os.splitFile(currentFilename.string).dir
let outDirPath: RelativeFile =
presentationPath(conf, AbsoluteFile(basedir / targetRelPath))
# use presentationPath because `..` path can be be mangled to `_._`
result.targetPath = string(conf.outDir / outDirPath)
if not fileExists(result.targetPath):
# this can happen if targetRelPath goes to parent directory `OUTDIR/..`.
# Trying it, this may cause ambiguities, but allows us to insert
# "packages" into each other, which is actually used in Nim repo itself.
let destPath = fromDir / targetRelPath
if destPath != result.targetPath and fileExists(destPath):
result.targetPath = destPath
result.linkRelPath = relativePath(result.targetPath.splitFile.dir,
fromDir).replace('\\', '/')
proc parseRst(text: string,
line, column: int,
conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
declareClosures()
result = rstParsePass1(text, line, column, sharedState)
proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
@ -283,7 +307,8 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
outExt: string = HtmlExt, module: PSym = nil,
standaloneDoc = false, preferMarkdown = true,
hasToc = true): PDoc =
declareClosures()
let destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
declareClosures(currentFilename = filename, destFile = destFile)
new(result)
result.module = module
result.conf = conf
@ -298,7 +323,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
result.hasToc = hasToc
result.sharedState = newRstSharedState(
options, filename.string,
docgenFindFile, compilerMsgHandler, hasToc)
docgenFindFile, docgenFindRefFile, compilerMsgHandler, hasToc)
initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
conf.configVars, filename.string,
docgenFindFile, compilerMsgHandler)
@ -373,7 +398,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
if gotten != status:
rawMessage(conf, errGenerated, "snippet failed: cmd: '$1' status: $2 expected: $3 output: $4" % [cmd, $gotten, $status, output])
result.emitted = initIntSet()
result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
result.destFile = destFile
result.thisDir = result.destFile.AbsoluteFile.splitFile.dir
template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
@ -765,21 +790,24 @@ proc isVisible(d: PDoc; n: PNode): bool =
elif n.kind == nkPragmaExpr:
result = isVisible(d, n[0])
proc getName(d: PDoc, n: PNode, splitAfter = -1): string =
proc getName(n: PNode): string =
case n.kind
of nkPostfix: result = getName(d, n[1], splitAfter)
of nkPragmaExpr: result = getName(d, n[0], splitAfter)
of nkSym: result = esc(d.target, n.sym.renderDefinitionName, splitAfter)
of nkIdent: result = esc(d.target, n.ident.s, splitAfter)
of nkPostfix: result = getName(n[1])
of nkPragmaExpr: result = getName(n[0])
of nkSym: result = n.sym.renderDefinitionName
of nkIdent: result = n.ident.s
of nkAccQuoted:
result = esc(d.target, "`")
for i in 0..<n.len: result.add(getName(d, n[i], splitAfter))
result.add esc(d.target, "`")
result = "`"
for i in 0..<n.len: result.add(getName(n[i]))
result = "`"
of nkOpenSymChoice, nkClosedSymChoice:
result = getName(d, n[0], splitAfter)
result = getName(n[0])
else:
result = ""
proc getNameEsc(d: PDoc, n: PNode): string =
esc(d.target, getName(n))
proc getNameIdent(cache: IdentCache; n: PNode): PIdent =
case n.kind
of nkPostfix: result = getNameIdent(cache, n[1])
@ -928,6 +956,13 @@ proc symbolPriority(k: TSymKind): int =
else: 0 # including skProc which have higher priority
# documentation itself has even higher priority 1
proc getTypeKind(n: PNode): string =
case n[2].kind
of nkEnumTy: "enum"
of nkObjectTy: "object"
of nkTupleTy: "tuple"
else: ""
proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
## Converts symbol info (names/types/parameters) in `n` into format
## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``.
@ -971,17 +1006,13 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
if kind != tkSpaces:
result.generics.add(literal.nimIdentNormalize)
if k == skType:
case n[2].kind
of nkEnumTy: result.symTypeKind = "enum"
of nkObjectTy: result.symTypeKind = "object"
of nkTupleTy: result.symTypeKind = "tuple"
else: discard
if k == skType: result.symTypeKind = getTypeKind(n)
proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
if (docFlags != kForceExport) and not isVisible(d, nameNode): return
let
name = getName(d, nameNode)
name = getName(nameNode)
nameEsc = esc(d.target, name)
var plainDocstring = getPlainDocstring(n) # call here before genRecComment!
var result = ""
var literal, plainName = ""
@ -1008,9 +1039,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
inc(d.id)
let
plainNameEsc = esc(d.target, plainName.strip)
detailedName = k.toHumanStr & " " & (
typeDescr =
if k == skType and getTypeKind(n) != "": getTypeKind(n)
else: k.toHumanStr
detailedName = typeDescr & " " & (
if k in routineKinds: plainName else: name)
uniqueName = if k in routineKinds: plainNameEsc else: name
uniqueName = if k in routineKinds: plainNameEsc else: nameEsc
sortName = if k in routineKinds: plainName.strip else: name
cleanPlainSymbol = renderPlainSymbolName(nameNode)
complexSymbol = complexName(k, n, cleanPlainSymbol)
@ -1024,8 +1058,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
let lineinfo = rstast.TLineInfo(
line: nameNode.info.line, col: nameNode.info.col,
fileIndex: addRstFileIndex(d, nameNode.info))
addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName,
rstLangSymbol, priority = symbolPriority(k), info = lineinfo)
addAnchorNim(d.sharedState, external = false, refn = symbolOrId,
tooltip = detailedName, langSym = rstLangSymbol,
priority = symbolPriority(k), info = lineinfo)
nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
renderDocComments, renderSyms, renderExpandUsing}, symbolOrIdEnc)
@ -1060,8 +1095,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
if e.sym.kind != skEnumField: continue
let plain = renderPlainSymbolName(e)
let symbolOrId = d.newUniquePlainSymbol(plain)
setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
xmltree.escape(getPlainDocstring(e).docstringSummary))
setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId,
term = plain, linkTitle = nameNode.sym.name.s & '.' & plain,
linkDesc = xmltree.escape(getPlainDocstring(e).docstringSummary),
line = n.info.line.int)
d.tocSimple[k].add TocItem(
sortName: sortName,
@ -1076,22 +1113,17 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
"itemSymOrID", symbolOrId.replace(",", ",<wbr>"),
"itemSymOrIDEnc", symbolOrIdEnc])
# Ironically for types the complexSymbol is *cleaner* than the plainName
# because it doesn't include object fields or documentation comments. So we
# use the plain one for callable elements, and the complex for the rest.
var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip))
else: linkTitle.add(xmltree.escape(complexSymbol.strip))
setIndexTerm(d[], external, symbolOrId, name, linkTitle,
xmltree.escape(plainDocstring.docstringSummary))
setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId, term = name,
linkTitle = detailedName,
linkDesc = xmltree.escape(plainDocstring.docstringSummary),
line = n.info.line.int)
if k == skType and nameNode.kind == nkSym:
d.types.strTableAdd nameNode.sym
proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
if not isVisible(d, nameNode): return
var
name = getName(d, nameNode)
name = getNameEsc(d, nameNode)
comm = genRecComment(d, n)
r: TSrcGen
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing})
@ -1337,9 +1369,31 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
proc overloadGroupName(s: string, k: TSymKind): string =
## Turns a name like `f` into anchor `f-procs-all`
#s & " " & k.toHumanStr & "s all"
s & "-" & k.toHumanStr & "s-all"
proc setIndexTitle(d: PDoc, useMetaTitle: bool) =
let titleKind = if d.standaloneDoc: ieMarkupTitle else: ieNimTitle
let external = AbsoluteFile(d.destFile)
.relativeTo(d.conf.outDir, '/')
.changeFileExt(HtmlExt)
.string
var term, linkTitle: string
if useMetaTitle and d.meta[metaTitle].len != 0:
term = d.meta[metaTitleRaw]
linkTitle = d.meta[metaTitleRaw]
else:
let filename = extractFilename(d.filename)
term =
if d.standaloneDoc: filename # keep .rst/.md extension
else: changeFileExt(filename, "") # rm .nim extension
linkTitle =
if d.standaloneDoc: term # keep .rst/.md extension
else: canonicalImport(d.conf, AbsoluteFile d.filename)
if not d.standaloneDoc:
linkTitle = "module " & linkTitle
setIndexTerm(d[], titleKind, htmlFile = external, id = "",
term = term, linkTitle = linkTitle)
proc finishGenerateDoc*(d: var PDoc) =
## Perform 2nd RST pass for resolution of links/footnotes/headings...
# copy file map `filenames` to ``rstgen.nim`` for its warnings
@ -1352,7 +1406,24 @@ proc finishGenerateDoc*(d: var PDoc) =
firstRst = fragment.rst
break
d.hasToc = d.hasToc or d.sharedState.hasToc
preparePass2(d.sharedState, firstRst)
# in --index:only mode we do NOT want to load other .idx, only write ours:
let importdoc = optGenIndexOnly notin d.conf.globalOptions and
optNoImportdoc notin d.conf.globalOptions
preparePass2(d.sharedState, firstRst, importdoc)
if optGenIndexOnly in d.conf.globalOptions:
# Top-level doc.comments may contain titles and :idx: statements:
for fragment in d.modDescPre:
if fragment.isRst:
traverseForIndex(d[], fragment.rst)
setIndexTitle(d, useMetaTitle = d.standaloneDoc)
# Symbol-associated doc.comments may contain :idx: statements:
for k in TSymKind:
for _, overloadChoices in d.section[k].secItems:
for item in overloadChoices:
for fragment in item.descRst:
if fragment.isRst:
traverseForIndex(d[], fragment.rst)
# add anchors to overload groups before RST resolution
for k in TSymKind:
@ -1362,14 +1433,25 @@ proc finishGenerateDoc*(d: var PDoc) =
let refn = overloadGroupName(plainName, k)
let tooltip = "$1 ($2 overloads)" % [
k.toHumanStr & " " & plainName, $overloadChoices.len]
addAnchorNim(d.sharedState, refn, tooltip,
let name = nimIdentBackticksNormalize(plainName)
# save overload group to ``.idx``
let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').
changeFileExt(HtmlExt).string
setIndexTerm(d[], ieNimGroup, htmlFile = external, id = refn,
term = name, linkTitle = k.toHumanStr,
linkDesc = "", line = overloadChoices[0].info.line.int)
if optGenIndexOnly in d.conf.globalOptions: continue
addAnchorNim(d.sharedState, external=false, refn, tooltip,
LangSymbol(symKind: k.toHumanStr,
name: nimIdentBackticksNormalize(plainName),
name: name,
isGroup: true),
priority = symbolPriority(k),
# select index `0` just to have any meaningful warning:
info = overloadChoices[0].info)
if optGenIndexOnly in d.conf.globalOptions:
return
# Finalize fragments of ``.nim`` or ``.rst`` file
proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
for f in fragments:
@ -1421,6 +1503,9 @@ proc finishGenerateDoc*(d: var PDoc) =
d.jEntriesFinal.add entry.json # generates docs
setIndexTitle(d, useMetaTitle = d.standaloneDoc)
completePass2(d.sharedState)
proc add(d: PDoc; j: JsonItem) =
if j.json != nil or j.rst != nil: d.jEntriesPre.add j
@ -1467,7 +1552,7 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
else: discard
proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string =
result = getName(d, nameNode) & "\n"
result = getNameEsc(d, nameNode) & "\n"
proc generateTags*(d: PDoc, n: PNode, r: var string) =
case n.kind
@ -1574,13 +1659,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
# Extract the title. Non API modules generate an entry in the index table.
if d.meta[metaTitle].len != 0:
title = d.meta[metaTitle]
let external = AbsoluteFile(d.destFile)
.relativeTo(d.conf.outDir, '/')
.changeFileExt(HtmlExt)
.string
setIndexTerm(d[], external, "", title)
else:
# Modules get an automatic title for the HTML, but no entry in the index.
title = canonicalImport(d.conf, AbsoluteFile d.filename)
title = esc(d.target, title)
var subtitle = ""
@ -1619,11 +1698,17 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
code = content
result = code
proc indexFile(d: PDoc): AbsoluteFile =
let dir = d.conf.outDir
result = dir / changeFileExt(presentationPath(d.conf,
AbsoluteFile d.filename),
IndexExt)
let (finalDir, _, _) = result.string.splitFile
createDir(finalDir)
proc generateIndex*(d: PDoc) =
if optGenIndex in d.conf.globalOptions:
let dir = d.conf.outDir
createDir(dir)
let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
let dest = indexFile(d)
writeIndexFile(d[], dest.string)
proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
@ -1634,6 +1719,9 @@ proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
if optGenIndexOnly in d.conf.globalOptions:
d.conf.outFile = indexFile(d).relativeTo(d.conf.outDir) # just for display
return
runAllExamples(d)
var content = genOutFile(d, groupedToc)
if optStdout in d.conf.globalOptions:
@ -1772,6 +1860,8 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
rawMessage(conf, errCannotOpenFile, filename.string)
proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
if optGenIndexOnly in conf.globalOptions:
return
var content = mergeIndexes(dir)
var outFile = outFile