docgen: implement doc link resolution in current module (#18642)

This commit is contained in:
Andrey Makarov 2021-10-28 20:20:52 +03:00 • committed by GitHub
commit 7ba2659f73
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 1833 additions and 143 deletions

View file

@ -114,7 +114,7 @@
## .. _`extra features`:
##
## Optional additional features, turned on by ``options: RstParseOption`` in
## `rstParse proc <#rstParse,string,string,int,int,bool,RstParseOptions,FindFileHandler,MsgHandler>`_:
## `proc rstParse`_:
##
## * emoji / smiley symbols
## * Markdown tables
@ -196,7 +196,7 @@
## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
import
os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils,
std/private/miscdollars, tables
from highlite import SourceLanguage, getSourceLanguage
@ -231,6 +231,7 @@ type
meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1",
mwRedefinitionOfLabel = "redefinition of label '$1'",
mwUnknownSubstitution = "unknown substitution '$1'",
mwAmbiguousLink = "ambiguous doc link $1",
mwBrokenLink = "broken link '$1'",
mwUnsupportedLanguage = "language '$1' not supported",
mwUnsupportedField = "field '$1' not supported",
@ -473,12 +474,42 @@ type
hasPeers: bool # has headings on the same level of hierarchy?
LevelMap = seq[LevelInfo] # Saves for each possible title adornment
# style its level in the current document.
SubstitutionKind = enum
rstSubstitution = "substitution",
hyperlinkAlias = "hyperlink alias",
implicitHyperlinkAlias = "implicitly-generated hyperlink alias"
Substitution = object
kind*: SubstitutionKind
key*: string
value*: PRstNode
AnchorSubst = tuple
mainAnchor: string
aliases: seq[string]
info*: TLineInfo # place where the substitution was defined
AnchorRule = enum
arInternalRst, ## For automatically generated RST anchors (from
## headings, footnotes, inline internal targets):
## case-insensitive, 1-space-significant (by RST spec)
arNim ## For anchors generated by ``docgen.rst``: Nim-style case
## sensitivity, etc. (see `proc normalizeNimName`_ for details)
arHyperlink, ## For links with manually set anchors in
## form `text <pagename.html#anchor>`_
RstAnchorKind = enum
manualDirectiveAnchor = "manual directive anchor",
manualInlineAnchor = "manual inline anchor",
footnoteAnchor = "footnote anchor",
headlineAnchor = "implicitly-generated headline anchor"
AnchorSubst = object
mainAnchor: ref string # A reference name that will be inserted directly
# into HTML/Latex. It's declared as `ref` because
# it can be shared between aliases.
info: TLineInfo # where the anchor was defined
priority: int
case kind: range[arInternalRst .. arNim]
of arInternalRst:
anchorType: RstAnchorKind
of arNim:
tooltip: string # displayed tooltip for Nim-generated anchors
langSym: LangSymbol
AnchorSubstTable = Table[string, seq[AnchorSubst]]
# use `seq` to account for duplicate anchors
FootnoteType = enum
fnManualNumber, # manually numbered footnote like [3]
fnAutoNumber, # auto-numbered footnote [#]
@ -505,7 +536,8 @@ type
currRoleKind: RstNodeKind # ... and its node kind
subs: seq[Substitution] # substitutions
refs*: seq[Substitution] # references
anchors*: seq[AnchorSubst] # internal target substitutions
anchors*: AnchorSubstTable
# internal target substitutions
lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#]
lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_
currFootnoteNumRef: int # ... their counter for `resolveSubs`
@ -518,7 +550,7 @@ type
findFile: FindFileHandler # How to find files.
filenames*: RstFileTable # map file name <-> FileIndex (for storing
# file names for warnings after 1st stage)
currFileIdx: FileIndex # current index in `filesnames`
currFileIdx*: FileIndex # current index in `filenames`
hasToc*: bool
PRstSharedState* = ref RstSharedState
@ -532,6 +564,7 @@ type
## in case of error/warning reporting to
## (relative) line/column of the token.
curAnchor*: string # variable to track latest anchor in s.anchors
curAnchorName*: string # corresponding name in human-readable format
EParseError* = object of ValueError
@ -590,13 +623,16 @@ proc whichRoleAux(sym: string): RstNodeKind =
proc len(filenames: RstFileTable): int = filenames.idxToFilename.len
proc setCurrFilename(s: PRstSharedState, file1: string) =
proc addFilename*(s: PRstSharedState, file1: string): FileIndex =
## Returns index of filename, adding it if it has not been used before
let nextIdx = s.filenames.len.FileIndex
let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
if v == nextIdx:
s.filenames.filenameToIdx[file1] = v
result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
if result == nextIdx:
s.filenames.filenameToIdx[file1] = result
s.filenames.idxToFilename.add file1
s.currFileIdx = v
proc setCurrFilename*(s: PRstSharedState, file1: string) =
s.currFileIdx = addFilename(s, file1)
proc getFilename(filenames: RstFileTable, fid: FileIndex): string =
doAssert(0 <= fid.int and fid.int < filenames.len,
@ -730,6 +766,8 @@ proc initParser(p: var RstParser, sharedState: PRstSharedState) =
p.s = sharedState
proc addNodesAux(n: PRstNode, result: var string) =
if n == nil:
return
if n.kind == rnLeaf:
result.add(n.text)
else:
@ -738,6 +776,11 @@ proc addNodesAux(n: PRstNode, result: var string) =
proc addNodes(n: PRstNode): string =
n.addNodesAux(result)
proc linkName(n: PRstNode): string =
## Returns a normalized reference name, see:
## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
n.addNodes.toLowerAscii
proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) =
template special(s) =
if b:
@ -804,15 +847,26 @@ proc findSub(s: PRstSharedState, n: PRstNode): int =
return i
result = -1
proc lineInfo(p: RstParser, iTok: int): TLineInfo =
result.col = int16(p.col + p.tok[iTok].col)
result.line = uint16(p.line + p.tok[iTok].line)
result.fileIndex = p.s.currFileIdx
proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
# TODO: we need this simplification because we don't preserve exact starting
# token of currently parsed element:
proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1)
proc setSub(p: var RstParser, key: string, value: PRstNode) =
var length = p.s.subs.len
for i in 0 ..< length:
if key == p.s.subs[i].key:
p.s.subs[i].value = value
return
p.s.subs.add(Substitution(key: key, value: value))
p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p)))
proc setRef(p: var RstParser, key: string, value: PRstNode) =
proc setRef(p: var RstParser, key: string, value: PRstNode,
refType: SubstitutionKind) =
var length = p.s.refs.len
for i in 0 ..< length:
if key == p.s.refs[i].key:
@ -820,37 +874,111 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) =
rstMessage(p, mwRedefinitionOfLabel, key)
p.s.refs[i].value = value
return
p.s.refs.add(Substitution(key: key, value: value))
p.s.refs.add(Substitution(kind: refType, key: key, value: value,
info: prevLineInfo(p)))
proc findRef(s: PRstSharedState, key: string): PRstNode =
proc findRef(s: PRstSharedState, key: string): seq[Substitution] =
for i in countup(0, high(s.refs)):
if key == s.refs[i].key:
return s.refs[i].value
result.add s.refs[i]
proc addAnchor(p: var RstParser, refn: string, reset: bool) =
## add anchor `refn` to anchor aliases and update last anchor ``curAnchor``
if p.curAnchor == "":
p.s.anchors.add (refn, @[refn])
# Ambiguity in links: we don't follow procedure of removing implicit targets
# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets
# Instead we just give explicit links a higher priority than to implicit ones
# and report ambiguities as warnings. Hopefully it is easy to remove
# ambiguities manually. Nim auto-generated links from ``docgen.nim``
# have lowest priority: 1 (for procs) and below for other symbol types.
proc refPriority(k: SubstitutionKind): int =
case k
of rstSubstitution: result = 8
of hyperlinkAlias: result = 7
of implicitHyperlinkAlias: result = 2
proc internalRefPriority(k: RstAnchorKind): int =
case k
of manualDirectiveAnchor: result = 6
of manualInlineAnchor: result = 5
of footnoteAnchor: result = 4
of headlineAnchor: result = 3
proc addAnchorRst(p: var RstParser, name: string, refn: string, reset: bool,
anchorType: RstAnchorKind) =
## Adds anchor `refn` with an alias `name` and
## updates the corresponding `curAnchor` / `curAnchorName`.
let prio = internalRefPriority(anchorType)
if p.curAnchorName == "":
var anchRef = new string
anchRef[] = refn
p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add(
AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio,
info: prevLineInfo(p), anchorType: anchorType))
else:
p.s.anchors[^1].mainAnchor = refn
p.s.anchors[^1].aliases.add refn
# override previous mainAnchor by `ref` in all aliases
var anchRef = p.s.anchors[p.curAnchorName][0].mainAnchor
anchRef[] = refn
p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add(
AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio,
info: prevLineInfo(p), anchorType: anchorType))
if reset:
p.curAnchor = ""
p.curAnchorName = ""
else:
p.curAnchor = refn
p.curAnchorName = name
proc findMainAnchor(s: PRstSharedState, refn: string): string =
for subst in s.anchors:
if subst.mainAnchor == refn: # no need to rename
result = subst.mainAnchor
break
var toLeave = false
for anchor in subst.aliases:
if anchor == refn: # this anchor will be named as mainAnchor
result = subst.mainAnchor
toLeave = true
if toLeave:
break
proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string,
langSym: LangSymbol, priority: int,
info: TLineInfo) =
## Adds an anchor `refn` (`mainAnchor`), which follows
## the rule `arNim` (i.e. a symbol in ``*.nim`` file)
var anchRef = new string
anchRef[] = refn
s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add(
AnchorSubst(kind: arNim, mainAnchor: anchRef, langSym: langSym,
tooltip: tooltip, priority: priority,
info: info))
proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode,
info: TLineInfo):
seq[AnchorSubst] =
let langSym = toLangSymbol(signature)
let substitutions = s.anchors.getOrDefault(langSym.name,
newSeq[AnchorSubst]())
if substitutions.len == 0:
return
# map symKind (like "proc") -> found symbols/groups:
var found: Table[string, seq[AnchorSubst]]
for s in substitutions:
if s.kind == arNim:
if match(s.langSym, langSym):
found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s
for symKind, sList in found:
if sList.len == 1:
result.add sList[0]
else: # > 1, there are overloads, potential ambiguity in this `symKind`
if langSym.parametersProvided:
# there are non-group signatures, select only them
for s in sList:
if not s.langSym.isGroup:
result.add s
else: # when there are many overloads a link like foo_ points to all
# of them, so selecting the group
var foundGroup = true
for s in sList:
if s.langSym.isGroup:
result.add s
foundGroup = true
break
doAssert foundGroup, "docgen has not generated the group"
proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo):
seq[AnchorSubst] =
let name = linkText.toLowerAscii
let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]())
for s in substitutions:
if s.kind == arInternalRst:
result.add s
proc addFootnoteNumManual(p: var RstParser, num: int) =
## add manually-numbered footnote
@ -860,13 +988,6 @@ proc addFootnoteNumManual(p: var RstParser, num: int) =
return
p.s.footnotes.add((fnManualNumber, num, -1, -1, $num))
proc lineInfo(p: RstParser, iTok: int): TLineInfo =
result.col = int16(p.col + p.tok[iTok].col)
result.line = uint16(p.line + p.tok[iTok].line)
result.fileIndex = p.s.currFileIdx
proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
proc addFootnoteNumAuto(p: var RstParser, label: string) =
## add auto-numbered footnote.
## Empty label [#] means it'll be resolved by the occurrence.
@ -989,6 +1110,7 @@ proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode =
if p.curAnchor != "":
result.anchor = p.curAnchor
p.curAnchor = ""
p.curAnchorName = ""
template newLeaf(s: string): PRstNode = newRstLeaf(s)
@ -1255,7 +1377,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
else:
newKind = rnHyperlink
newSons = @[a, b]
setRef(p, rstnodeToRefname(a), b)
setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias)
result = newRstNode(newKind, newSons)
else: # some link that will be resolved in `resolveSubs`
newKind = rnRef
@ -1562,7 +1684,8 @@ proc parseInline(p: var RstParser, father: PRstNode) =
inc p.idx
parseUntil(p, n, "`", false)
let refn = rstnodeToRefname(n)
p.s.anchors.add (refn, @[refn])
addAnchorRst(p, name = linkName(n), refn = refn, reset = true,
anchorType=manualInlineAnchor)
father.add(n)
elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
inc p.idx
@ -2084,7 +2207,8 @@ proc parseHeadline(p: var RstParser): PRstNode =
result.level = getLevel(p, c, hasOverline=false)
checkHeadingHierarchy(p, result.level)
p.s.hCurLevel = result.level
addAnchor(p, rstnodeToRefname(result), reset=true)
addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true,
anchorType=headlineAnchor)
proc parseOverline(p: var RstParser): PRstNode =
var c = currentTok(p).symbol[0]
@ -2106,7 +2230,8 @@ proc parseOverline(p: var RstParser): PRstNode =
if currentTok(p).kind == tkAdornment:
inc p.idx
if currentTok(p).kind == tkIndent: inc p.idx
addAnchor(p, rstnodeToRefname(result), reset=true)
addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true,
anchorType=headlineAnchor)
type
IntSeq = seq[int]
@ -2837,7 +2962,7 @@ proc parseFootnote(p: var RstParser): PRstNode =
anchor.add $p.s.lineFootnoteSym.len
of fnCitation:
anchor.add rstnodeToRefname(label)
addAnchor(p, anchor, reset=true)
addAnchorRst(p, anchor, anchor, reset=true, anchorType=footnoteAnchor)
result.anchor = anchor
if currentTok(p).kind == tkWhite: inc p.idx
discard parseBlockContent(p, result, parseSectionWrapper)
@ -2858,13 +2983,23 @@ proc parseDotDot(p: var RstParser): PRstNode =
elif match(p, p.idx, " _"):
# hyperlink target:
inc p.idx, 2
var a = getReferenceName(p, ":")
var ending = ":"
if currentTok(p).symbol == "`":
inc p.idx
ending = "`"
var a = getReferenceName(p, ending)
if ending == "`":
if currentTok(p).symbol == ":":
inc p.idx
else:
rstMessage(p, meExpected, ":")
if currentTok(p).kind == tkWhite: inc p.idx
var b = untilEol(p)
if len(b) == 0: # set internal anchor
addAnchor(p, rstnodeToRefname(a), reset=false)
addAnchorRst(p, linkName(a), rstnodeToRefname(a), reset=false,
anchorType=manualDirectiveAnchor)
else: # external hyperlink
setRef(p, rstnodeToRefname(a), b)
setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias)
elif match(p, p.idx, " |"):
# substitution definitions:
inc p.idx, 2
@ -2892,7 +3027,7 @@ proc rstParsePass1*(fragment: string,
sharedState: PRstSharedState): PRstNode =
## Parses an RST `fragment`.
## The result should be further processed by
## `preparePass2` and `resolveSubs` (which is pass 2).
## preparePass2_ and resolveSubs_ (which is pass 2).
var p: RstParser
initParser(p, sharedState)
p.line = line
@ -2905,6 +3040,65 @@ proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) =
countTitles(s, mainNode)
orderFootnotes(s)
proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode =
# Associate this link alias with its target and change node kind to
# rnHyperlink or rnInternalRef appropriately.
type LinkDef = object
ar: AnchorRule
priority: int
tooltip: string
target: PRstNode
info: TLineInfo
proc cmp(x, y: LinkDef): int =
result = cmp(x.priority, y.priority)
if result == 0:
result = cmp(x.target, y.target)
var foundLinks: seq[LinkDef]
let text = newRstNode(rnInner, n.sons)
let refn = rstnodeToRefname(n)
var hyperlinks = findRef(s, refn)
for y in hyperlinks:
foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind),
target: y.value, info: y.info,
tooltip: "(" & $y.kind & ")")
let substRst = findMainAnchorRst(s, text.addNodes, n.info)
for subst in substRst:
foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority,
target: newLeaf(subst.mainAnchor[]),
info: subst.info,
tooltip: "(" & $subst.anchorType & ")")
if roNimFile in s.options:
let substNim = findMainAnchorNim(s, signature=text, n.info)
for subst in substNim:
foundLinks.add LinkDef(ar: arNim, priority: subst.priority,
target: newLeaf(subst.mainAnchor[]),
info: subst.info, tooltip: subst.tooltip)
foundLinks.sort(cmp = cmp, order = Descending)
let linkText = addNodes(n)
if foundLinks.len >= 1:
let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink
elif foundLinks[0].ar == arNim: rnNimdocRef
else: rnInternalRef
result = newRstNode(kind)
result.sons = @[text, foundLinks[0].target]
if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip
if foundLinks.len > 1: # report ambiguous link
var targets = newSeq[string]()
for l in foundLinks:
var t = " "
if s.filenames.len > 1:
t.add getFilename(s.filenames, l.info.fileIndex)
let n = l.info.line
let c = l.info.col + ColRstOffset
t.add "($1, $2): $3" % [$n, $c, l.tooltip]
targets.add t
rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink,
"`$1`\n clash:\n$2" % [
linkText, targets.join("\n")])
else: # nothing found
result = n
rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, linkText)
proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
## Makes pass 2 of RST parsing.
## Resolves substitutions and anchor aliases, groups footnotes.
@ -2933,21 +3127,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
elif s.hTitleCnt == 0:
n.level += 1
of rnRef:
let refn = rstnodeToRefname(n)
var y = findRef(s, refn)
if y != nil:
result = newRstNode(rnHyperlink)
let text = newRstNode(rnInner, n.sons)
result.sons = @[text, y]
else:
let anchor = findMainAnchor(s, refn)
if anchor != "":
result = newRstNode(rnInternalRef)
let text = newRstNode(rnInner, n.sons)
result.sons = @[text, # visible text of reference
newLeaf(anchor)] # link itself
else:
rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
result = resolveLink(s, n)
of rnFootnote:
var (fnType, num) = getFootnoteType(n.sons[0])
case fnType
@ -2993,9 +3173,10 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
of fnCitation:
result.add n.sons[0]
refn.add rstnodeToRefname(n)
let anch = findMainAnchor(s, refn)
if anch != "":
result.add newLeaf(anch) # add link
# TODO: correctly report ambiguities
let anchorInfo = findMainAnchorRst(s, refn, n.info)
if anchorInfo.len != 0:
result.add newLeaf(anchorInfo[0].mainAnchor[]) # add link
else:
rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
result.add newLeaf(refn) # add link