jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

docgen: move to shared RST state (fix #16990) (#18256)

Browse source 
* docgen: move to shared RST state (fix #16990)

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* rename `cmdDoc2` to `cmdDoc`

* fix (P)RstSharedState convention

* new style of initialization

* misc suggestions

* 1 more rename

* fix a regression

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>
Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>
This commit is contained in:
Andrey Makarov 2021-06-20 10:50:03 +03:00 • committed by GitHub
commit 590d457631
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 292 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/commands.nim
View file

@ -443,7 +443,7 @@ proc parseCommand*(command: string): Command =
of "check": cmdCheck of "check": cmdCheck
of "e": cmdNimscript of "e": cmdNimscript
of "doc0": cmdDoc0 of "doc0": cmdDoc0
of "doc2", "doc": cmdDoc2 of "doc2", "doc": cmdDoc
of "doc2tex": cmdDoc2tex of "doc2tex": cmdDoc2tex
of "rst2html": cmdRst2html of "rst2html": cmdRst2html
of "rst2tex": cmdRst2tex of "rst2tex": cmdRst2tex

241
compiler/docgen.nim
View file

@ -28,6 +28,24 @@ const
DocColOffset = "## ".len # assuming that a space was added after ## DocColOffset = "## ".len # assuming that a space was added after ##
type type
ItemFragment = object ## A fragment from each item will be eventually
## constructed by converting `rst` fields to strings.
case isRst: bool
of true:
rst: PRstNode
of false: ## contains ready markup e.g. from runnableExamples
str: string
ItemPre = seq[ItemFragment] ## A pre-processed item.
Item = object ## Any item in documentation, e.g. symbol
## entry. Configuration variable ``doc.item``
## is used for its HTML rendering.
descRst: ItemPre ## Description of the item (may contain
## runnableExamples).
substitutions: seq[string] ## Variable names in `doc.item`...
ModSection = object ## Section like Procs, Types, etc.
secItems: seq[Item] ## Pre-processed items.
finalMarkup: string ## The items, after RST pass 2 and rendering.
ModSections = array[TSymKind, ModSection]
TSections = array[TSymKind, string] TSections = array[TSymKind, string]
ExampleGroup = ref object ExampleGroup = ref object
## a group of runnableExamples with same rdoccmd ## a group of runnableExamples with same rdoccmd
@ -35,17 +53,25 @@ type
docCmd: string ## from user config, e.g. --doccmd:-d:foo docCmd: string ## from user config, e.g. --doccmd:-d:foo
code: string ## contains imports; each import contains `body` code: string ## contains imports; each import contains `body`
index: int ## group index index: int ## group index
JsonItem = object # pre-processed item: `rst` should be finalized
json: JsonNode
rst: PRstNode
rstField: string
TDocumentor = object of rstgen.RstGenerator TDocumentor = object of rstgen.RstGenerator
modDesc: string # module description modDescPre: ItemPre # module description, not finalized
modDescFinal: string # module description, after RST pass 2 and rendering
module: PSym module: PSym
modDeprecationMsg: string modDeprecationMsg: string
toc, toc2, section: TSections section: ModSections # entries of ``.nim`` file (for `proc`s, etc)
toc, toc2: TSections # toc2 - grouped TOC
tocTable: array[TSymKind, Table[string, string]] tocTable: array[TSymKind, Table[string, string]]
indexValFilename: string indexValFilename: string
analytics: string # Google Analytics javascript, "" if doesn't exist analytics: string # Google Analytics javascript, "" if doesn't exist
seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
jArray: JsonNode jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content
jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering
types: TStrTable types: TStrTable
sharedState: PRstSharedState
isPureRst: bool isPureRst: bool
conf*: ConfigRef conf*: ConfigRef
cache*: IdentCache cache*: IdentCache
@ -58,6 +84,9 @@ type
PDoc* = ref TDocumentor ## Alias to type less. PDoc* = ref TDocumentor ## Alias to type less.
proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
proc prettyString(a: object): string = proc prettyString(a: object): string =
# xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906 # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906
for k, v in fieldPairs(a): for k, v in fieldPairs(a):
@ -151,12 +180,12 @@ template declareClosures =
if not fileExists(result): result = "" if not fileExists(result): result = ""
proc parseRst(text, filename: string, proc parseRst(text, filename: string,
line, column: int, hasToc: var bool, line, column: int,
rstOptions: RstParseOptions; rstOptions: RstParseOptions;
conf: ConfigRef): PRstNode = conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
declareClosures() declareClosures()
result = rstParse(text, filename, line, column, hasToc, rstOptions, result = rstParsePass1(text, filename, line, column, rstOptions,
docgenFindFile, compilerMsgHandler) sharedState)
proc getOutFile2(conf: ConfigRef; filename: RelativeFile, proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
ext: string, guessTarget: bool): AbsoluteFile = ext: string, guessTarget: bool): AbsoluteFile =
@ -179,10 +208,13 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
result.conf = conf result.conf = conf
result.cache = cache result.cache = cache
result.outDir = conf.outDir.string result.outDir = conf.outDir.string
const options = {roSupportRawDirective, roSupportMarkdown,
roPreferMarkdown, roNimFile}
result.sharedState = newRstSharedState(
options, filename.string,
docgenFindFile, compilerMsgHandler)
initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
conf.configVars, filename.string, conf.configVars, filename.string, options,
{roSupportRawDirective, roSupportMarkdown,
roPreferMarkdown, roNimFile},
docgenFindFile, compilerMsgHandler) docgenFindFile, compilerMsgHandler)
if conf.configVars.hasKey("doc.googleAnalytics"): if conf.configVars.hasKey("doc.googleAnalytics"):
@ -203,7 +235,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
result.seenSymbols = newStringTable(modeCaseInsensitive) result.seenSymbols = newStringTable(modeCaseInsensitive)
result.id = 100 result.id = 100
result.jArray = newJArray() result.jEntriesFinal = newJArray()
initStrTable result.types initStrTable result.types
result.onTestSnippet = result.onTestSnippet =
proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
@ -261,37 +293,31 @@ proc getVarIdx(varnames: openArray[string], id: string): int =
return i return i
result = -1 result = -1
proc genComment(d: PDoc, n: PNode): string = proc genComment(d: PDoc, n: PNode): PRstNode =
result = ""
if n.comment.len > 0: if n.comment.len > 0:
let comment = n.comment result = parseRst(n.comment, toFullPath(d.conf, n.info),
when false: toLinenumber(n.info),
# RFC: to preseve newlines in comments, this would work: toColumn(n.info) + DocColOffset,
comment = comment.replace("\n", "\n\n") d.options, d.conf,
renderRstToOut(d[], d.sharedState)
parseRst(comment, toFullPath(d.conf, n.info),
toLinenumber(n.info),
toColumn(n.info) + DocColOffset,
(var dummy: bool; dummy), d.options, d.conf),
result)
proc genRecCommentAux(d: PDoc, n: PNode): string = proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
if n == nil: return "" if n == nil: return nil
result = genComment(d, n) result = genComment(d, n)
if result == "": if result == nil:
if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef,
nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}: nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}:
# notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}:
for i in 0..<n.len: for i in 0..<n.len:
result = genRecCommentAux(d, n[i]) result = genRecCommentAux(d, n[i])
if result != "": return if result != nil: return
else: else:
n.comment = "" n.comment = ""
proc genRecComment(d: PDoc, n: PNode): string = proc genRecComment(d: PDoc, n: PNode): PRstNode =
if n == nil: return "" if n == nil: return nil
result = genComment(d, n) result = genComment(d, n)
if result == "": if result == nil:
if n.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, if n.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
nkMacroDef, nkTemplateDef, nkConverterDef}: nkMacroDef, nkTemplateDef, nkConverterDef}:
result = genRecCommentAux(d, n[bodyPos]) result = genRecCommentAux(d, n[bodyPos])
@ -519,7 +545,7 @@ type RunnableState = enum
rsRunnable rsRunnable
rsDone rsDone
proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string, proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var ItemPre,
state: RunnableState, topLevel: bool): state: RunnableState, topLevel: bool):
RunnableState = RunnableState =
##[ ##[
@ -550,8 +576,10 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
let (rdoccmd, code) = prepareExample(d, n, topLevel) let (rdoccmd, code) = prepareExample(d, n, topLevel)
var msg = "Example:" var msg = "Example:"
if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n", var s: string
dispA(d.conf, s, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
"\n\n\\textbf{$1}\n", [msg]) "\n\n\\textbf{$1}\n", [msg])
dest.add s
inc d.listingCounter inc d.listingCounter
let id = $d.listingCounter let id = $d.listingCounter
dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
@ -597,7 +625,7 @@ proc getRoutineBody(n: PNode): PNode =
doAssert result.len == 2 doAssert result.len == 2
result = result[1] result = result[1]
proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var string) = proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) =
var n = n var n = n
var state = rsStart var state = rsStart
template fn(n2, topLevel) = template fn(n2, topLevel) =
@ -706,7 +734,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
## types will be added with a preceding dash. Return types won't be added. ## types will be added with a preceding dash. Return types won't be added.
## ##
## If you modify the output of this proc, please update the anchor generation ## If you modify the output of this proc, please update the anchor generation
## section of ``doc/docgen.txt``. ## section of ``doc/docgen.rst``.
result = baseName result = baseName
case k case k
of skProc, skFunc: discard of skProc, skFunc: discard
@ -804,7 +832,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
var result = "" var result = ""
var literal, plainName = "" var literal, plainName = ""
var kind = tkEof var kind = tkEof
var comm = "" var comm: ItemPre
if n.kind in routineDefs: if n.kind in routineDefs:
getAllRunnableExamples(d, n, comm) getAllRunnableExamples(d, n, comm)
else: else:
@ -840,9 +868,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
d.section[k].add(getConfigVar(d.conf, "doc.item") % d.section[k].secItems.add Item(descRst: comm, substitutions: @[
["name", name, "uniqueName", uniqueName, "name", name, "uniqueName", uniqueName,
"header", result, "desc", comm, "itemID", $d.id, "header", result, "itemID", $d.id,
"header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
"itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
"itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@ -886,25 +914,26 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
if k == skType and nameNode.kind == nkSym: if k == skType and nameNode.kind == nkSym:
d.types.strTableAdd nameNode.sym d.types.strTableAdd nameNode.sym
proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem =
if not isVisible(d, nameNode): return if not isVisible(d, nameNode): return
var var
name = getName(d, nameNode) name = getName(d, nameNode)
comm = genRecComment(d, n) comm = genRecComment(d, n)
r: TSrcGen r: TSrcGen
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments})
result = %{ "name": %name, "type": %($k), "line": %n.info.line.int, result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int,
"col": %n.info.col} "col": %n.info.col}
if comm.len > 0: if comm != nil:
result["description"] = %comm result.rst = comm
result.rstField = "description"
if r.buf.len > 0: if r.buf.len > 0:
result["code"] = %r.buf result.json["code"] = %r.buf
if k in routineKinds: if k in routineKinds:
result["signature"] = newJObject() result.json["signature"] = newJObject()
if n[paramsPos][0].kind != nkEmpty: if n[paramsPos][0].kind != nkEmpty:
result["signature"]["return"] = %($n[paramsPos][0]) result.json["signature"]["return"] = %($n[paramsPos][0])
if n[paramsPos].len > 1: if n[paramsPos].len > 1:
result["signature"]["arguments"] = newJArray() result.json["signature"]["arguments"] = newJArray()
for paramIdx in 1 ..< n[paramsPos].len: for paramIdx in 1 ..< n[paramsPos].len:
for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2: for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2:
let let
@ -912,22 +941,22 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode =
paramType = $n[paramsPos][paramIdx][^2] paramType = $n[paramsPos][paramIdx][^2]
if n[paramsPos][paramIdx][^1].kind != nkEmpty: if n[paramsPos][paramIdx][^1].kind != nkEmpty:
let paramDefault = $n[paramsPos][paramIdx][^1] let paramDefault = $n[paramsPos][paramIdx][^1]
result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault}
else: else:
result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType}
if n[pragmasPos].kind != nkEmpty: if n[pragmasPos].kind != nkEmpty:
result["signature"]["pragmas"] = newJArray() result.json["signature"]["pragmas"] = newJArray()
for pragma in n[pragmasPos]: for pragma in n[pragmasPos]:
result["signature"]["pragmas"].add %($pragma) result.json["signature"]["pragmas"].add %($pragma)
if n[genericParamsPos].kind != nkEmpty: if n[genericParamsPos].kind != nkEmpty:
result["signature"]["genericParams"] = newJArray() result.json["signature"]["genericParams"] = newJArray()
for genericParam in n[genericParamsPos]: for genericParam in n[genericParamsPos]:
var param = %{"name": %($genericParam)} var param = %{"name": %($genericParam)}
if genericParam.sym.typ.sons.len > 0: if genericParam.sym.typ.sons.len > 0:
param["types"] = newJArray() param["types"] = newJArray()
for kind in genericParam.sym.typ.sons: for kind in genericParam.sym.typ.sons:
param["types"].add %($kind) param["types"].add %($kind)
result["signature"]["genericParams"].add param result.json["signature"]["genericParams"].add param
proc checkForFalse(n: PNode): bool = proc checkForFalse(n: PNode): bool =
result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0
@ -946,8 +975,8 @@ proc traceDeps(d: PDoc, it: PNode) =
traceDeps(d, a) traceDeps(d, a)
elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
let external = externalDep(d, it.sym) let external = externalDep(d, it.sym)
if d.section[k] != "": d.section[k].add(", ") if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k], dispA(d.conf, d.section[k].finalMarkup,
"<a class=\"reference external\" href=\"$2\">$1</a>", "<a class=\"reference external\" href=\"$2\">$1</a>",
"$1", [esc(d.target, external.prettyLink), "$1", [esc(d.target, external.prettyLink),
changeFileExt(external, "html")]) changeFileExt(external, "html")])
@ -956,8 +985,8 @@ proc exportSym(d: PDoc; s: PSym) =
const k = exportSection const k = exportSection
if s.kind == skModule and belongsToPackage(d.conf, s): if s.kind == skModule and belongsToPackage(d.conf, s):
let external = externalDep(d, s) let external = externalDep(d, s)
if d.section[k] != "": d.section[k].add(", ") if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
dispA(d.conf, d.section[k], dispA(d.conf, d.section[k].finalMarkup,
"<a class=\"reference external\" href=\"$2\">$1</a>", "<a class=\"reference external\" href=\"$2\">$1</a>",
"$1", [esc(d.target, external.prettyLink), "$1", [esc(d.target, external.prettyLink),
changeFileExt(external, "html")]) changeFileExt(external, "html")])
@ -968,9 +997,9 @@ proc exportSym(d: PDoc; s: PSym) =
complexSymbol = complexName(s.kind, s.ast, s.name.s) complexSymbol = complexName(s.kind, s.ast, s.name.s)
symbolOrId = d.newUniquePlainSymbol(complexSymbol) symbolOrId = d.newUniquePlainSymbol(complexSymbol)
external = externalDep(d, module) external = externalDep(d, module)
if d.section[k] != "": d.section[k].add(", ") if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
# XXX proper anchor generation here # XXX proper anchor generation here
dispA(d.conf, d.section[k], dispA(d.conf, d.section[k].finalMarkup,
"<a href=\"$2#$3\"><span class=\"Identifier\">$1</span></a>", "<a href=\"$2#$3\"><span class=\"Identifier\">$1</span></a>",
"$1", [esc(d.target, s.name.s), "$1", [esc(d.target, s.name.s),
changeFileExt(external, "html"), changeFileExt(external, "html"),
@ -1034,13 +1063,16 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
if p5 != nil: n[pragmasPos].add p5 if p5 != nil: n[pragmasPos].add p5
proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
## Goes through nim nodes recursively and collects doc comments.
## Main function for `doc`:option: command,
## which is implemented in ``docgen2.nim``.
template genItemAux(skind) = template genItemAux(skind) =
genItem(d, n, n[namePos], skind, docFlags) genItem(d, n, n[namePos], skind, docFlags)
case n.kind case n.kind
of nkPragma: of nkPragma:
let pragmaNode = findPragma(n, wDeprecated) let pragmaNode = findPragma(n, wDeprecated)
d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode)) d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode))
of nkCommentStmt: d.modDesc.add(genComment(d, n)) of nkCommentStmt: d.modDescPre.add(genComment(d, n))
of nkProcDef, nkFuncDef: of nkProcDef, nkFuncDef:
when useEffectSystem: documentRaises(d.cache, n) when useEffectSystem: documentRaises(d.cache, n)
genItemAux(skProc) genItemAux(skProc)
@ -1079,21 +1111,63 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept" of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0]) of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0])
of nkCallKinds: of nkCallKinds:
var comm = "" var comm: ItemPre
getAllRunnableExamples(d, n, comm) getAllRunnableExamples(d, n, comm)
if comm != "": d.modDesc.add(comm) if comm.len != 0: d.modDescPre.add(comm)
else: discard else: discard
proc add(d: PDoc; j: JsonNode) = proc finishGenerateDoc*(d: var PDoc) =
if j != nil: d.jArray.add j ## Perform 2nd RST pass for resolution of links/footnotes/headings...
# Main title/subtitle are allowed only in the first RST fragment of document
var firstRst = PRstNode(nil)
for fragment in d.modDescPre:
if fragment.isRst:
firstRst = fragment.rst
break
preparePass2(d.sharedState, firstRst)
# Finalize fragments of ``.nim`` or ``.rst`` file
proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) =
for f in fragments:
case f.isRst:
of true:
var resolved = resolveSubs(d.sharedState, f.rst)
renderRstToOut(d[], resolved, result)
of false: result &= f.str
for k in TSymKind:
for item in d.section[k].secItems:
var itemDesc: string
renderItemPre(d, item.descRst, itemDesc)
d.section[k].finalMarkup.add(
getConfigVar(d.conf, "doc.item") % (
item.substitutions & @["desc", itemDesc]))
itemDesc = ""
d.section[k].secItems.setLen 0
renderItemPre(d, d.modDescPre, d.modDescFinal)
d.modDescPre.setLen 0
d.hasToc = d.hasToc or d.sharedState.hasToc
# Finalize fragments of ``.json`` file
for i, entry in d.jEntriesPre:
if entry.rst != nil:
let resolved = resolveSubs(d.sharedState, entry.rst)
var str: string
renderRstToOut(d[], resolved, str)
entry.json[entry.rstField] = %str
d.jEntriesFinal.add entry.json
d.jEntriesPre[i].rst = nil
proc add(d: PDoc; j: JsonItem) =
if j.json != nil or j.rst != nil: d.jEntriesPre.add j
proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) =
case n.kind case n.kind
of nkCommentStmt: of nkCommentStmt:
if includeComments: if includeComments:
d.add %*{"comment": genComment(d, n)} d.add JsonItem(rst: genComment(d, n), rstField: "comment",
json: %Table[string, string]())
else: else:
d.modDesc.add(genComment(d, n)) d.modDescPre.add(genComment(d, n))
of nkProcDef, nkFuncDef: of nkProcDef, nkFuncDef:
when useEffectSystem: documentRaises(d.cache, n) when useEffectSystem: documentRaises(d.cache, n)
d.add genJsonItem(d, n, n[namePos], skProc) d.add genJsonItem(d, n, n[namePos], skProc)
@ -1173,11 +1247,11 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
"Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs", "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
"Methods", "Iterators", "Converters", "Macros", "Templates", "Exports" "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports"
] ]
if d.section[kind] == "": return if d.section[kind].finalMarkup == "": return
var title = sectionNames[kind] var title = sectionNames[kind]
d.section[kind] = getConfigVar(d.conf, "doc.section") % [ d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [
"sectionid", $ord(kind), "sectionTitle", title, "sectionid", $ord(kind), "sectionTitle", title,
"sectionTitleID", $(ord(kind) + 50), "content", d.section[kind]] "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup]
var tocSource = d.toc var tocSource = d.toc
if groupedToc: if groupedToc:
@ -1209,7 +1283,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
if toc != "" or d.target == outLatex: if toc != "" or d.target == outLatex:
# for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
toc = getConfigVar(d.conf, "doc.toc") % ["content", toc] toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
for i in TSymKind: code.add(d.section[i]) for i in TSymKind: code.add(d.section[i].finalMarkup)
# Extract the title. Non API modules generate an entry in the index table. # Extract the title. Non API modules generate an entry in the index table.
if d.meta[metaTitle].len != 0: if d.meta[metaTitle].len != 0:
@ -1234,7 +1308,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
let seeSrc = genSeeSrc(d, d.filename, 1) let seeSrc = genSeeSrc(d, d.filename, 1)
content = getConfigVar(d.conf, bodyname) % [ content = getConfigVar(d.conf, bodyname) % [
"title", title, "subtitle", subtitle, "title", title, "subtitle", subtitle,
"tableofcontents", toc, "moduledesc", d.modDesc, "date", getDateStr(), "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(),
"time", getClockStr(), "content", code, "time", getClockStr(), "content", code,
"deprecationMsg", d.modDeprecationMsg, "deprecationMsg", d.modDeprecationMsg,
"theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile, "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
@ -1248,7 +1322,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
"dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile, "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile,
docHackJsFname.RelativeFile), docHackJsFname.RelativeFile),
"title", title, "subtitle", subtitle, "tableofcontents", toc, "title", title, "subtitle", subtitle, "tableofcontents", toc,
"moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(), "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(),
"content", content, "author", d.meta[metaAuthor], "content", content, "author", d.meta[metaAuthor],
"version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics, "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics,
"deprecationMsg", d.modDeprecationMsg] "deprecationMsg", d.modDeprecationMsg]
@ -1297,12 +1371,12 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
proc writeOutputJson*(d: PDoc, useWarning = false) = proc writeOutputJson*(d: PDoc, useWarning = false) =
runAllExamples(d) runAllExamples(d)
var modDesc: string var modDesc: string
for desc in d.modDesc: for desc in d.modDescFinal:
modDesc &= desc modDesc &= desc
let content = %*{"orig": d.filename, let content = %*{"orig": d.filename,
"nimble": getPackageName(d.conf, d.filename), "nimble": getPackageName(d.conf, d.filename),
"moduleDescription": modDesc, "moduleDescription": modDesc,
"entries": d.jArray} "entries": d.jEntriesFinal}
if optStdout in d.conf.globalOptions: if optStdout in d.conf.globalOptions:
write(stdout, $content) write(stdout, $content)
else: else:
@ -1324,12 +1398,14 @@ proc handleDocOutputOptions*(conf: ConfigRef) =
conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
proc commandDoc*(cache: IdentCache, conf: ConfigRef) = proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
## implementation of deprecated ``doc0`` command (without semantic checking)
handleDocOutputOptions conf handleDocOutputOptions conf
var ast = parseFile(conf.projectMainIdx, cache, conf) var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf) var d = newDocumentor(conf.projectFull, cache, conf)
d.hasToc = true d.hasToc = true
generateDoc(d, ast, ast) generateDoc(d, ast, ast)
finishGenerateDoc(d)
writeOutput(d) writeOutput(d)
generateIndex(d) generateIndex(d)
@ -1339,14 +1415,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
var d = newDocumentor(filen, cache, conf, outExt) var d = newDocumentor(filen, cache, conf, outExt)
d.isPureRst = true d.isPureRst = true
var rst = parseRst(readFile(filen.string), filen.string, let rst = parseRst(readFile(filen.string), filen.string,
line=LineRstInit, column=ColRstInit, line=LineRstInit, column=ColRstInit,
d.hasToc, {roSupportRawDirective, roSupportMarkdown,
{roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}, roPreferMarkdown}, conf,
conf) d.sharedState)
var modDesc = newStringOfCap(30_000) d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
renderRstToOut(d[], rst, modDesc) finishGenerateDoc(d)
d.modDesc = modDesc
writeOutput(d) writeOutput(d)
generateIndex(d) generateIndex(d)
@ -1357,6 +1432,7 @@ proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
commandRstAux(cache, conf, conf.projectFull, TexExt) commandRstAux(cache, conf, conf.projectFull, TexExt)
proc commandJson*(cache: IdentCache, conf: ConfigRef) = proc commandJson*(cache: IdentCache, conf: ConfigRef) =
## implementation of a deprecated jsondoc0 command
var ast = parseFile(conf.projectMainIdx, cache, conf) var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf) var d = newDocumentor(conf.projectFull, cache, conf)
@ -1366,7 +1442,8 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) =
warnUser, "the ':test:' attribute is not supported by this backend") warnUser, "the ':test:' attribute is not supported by this backend")
d.hasToc = true d.hasToc = true
generateJson(d, ast) generateJson(d, ast)
let json = d.jArray finishGenerateDoc(d)
let json = d.jEntriesFinal
let content = pretty(json) let content = pretty(json)
if optStdout in d.conf.globalOptions: if optStdout in d.conf.globalOptions:

1
compiler/docgen2.nim
View file

@ -31,6 +31,7 @@ template closeImpl(body: untyped) {.dirty.} =
let useWarning = sfMainModule notin g.module.flags let useWarning = sfMainModule notin g.module.flags
let groupedToc = true let groupedToc = true
if shouldProcess(g): if shouldProcess(g):
finishGenerateDoc(g.doc)
body body
try: try:
generateIndex(g.doc) generateIndex(g.doc)

2
compiler/main.nim
View file

@ -282,7 +282,7 @@ proc mainCommand*(graph: ModuleGraph) =
else: else:
rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc") rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
of cmdDoc0: docLikeCmd commandDoc(cache, conf) of cmdDoc0: docLikeCmd commandDoc(cache, conf)
of cmdDoc2: of cmdDoc:
docLikeCmd(): docLikeCmd():
conf.setNoteDefaults(warnLockLevel, false) # issue #13218 conf.setNoteDefaults(warnLockLevel, false) # issue #13218
conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218

6
compiler/options.nim
View file

@ -143,8 +143,8 @@ type
cmdIdeTools # ide tools (e.g. nimsuggest) cmdIdeTools # ide tools (e.g. nimsuggest)
cmdNimscript # evaluate nimscript cmdNimscript # evaluate nimscript
cmdDoc0 cmdDoc0
cmdDoc2 cmdDoc # convert .nim doc comments to HTML
cmdDoc2tex cmdDoc2tex # convert .nim doc comments to LaTeX
cmdRst2html # convert a reStructuredText file to HTML cmdRst2html # convert a reStructuredText file to HTML
cmdRst2tex # convert a reStructuredText file to TeX cmdRst2tex # convert a reStructuredText file to TeX
cmdJsondoc0 cmdJsondoc0
@ -161,7 +161,7 @@ type
const const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun} cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
cmdCtags, cmdBuildindex} cmdCtags, cmdBuildindex}
type type

215
lib/packages/docutils/rst.nim
View file

@ -441,7 +441,7 @@ proc rawGetTok(L: var Lexer, tok: var Token) =
inc L.col inc L.col
tok.col = max(tok.col - L.baseIndent, 0) tok.col = max(tok.col - L.baseIndent, 0)
proc getTokens(buffer: string, tokens: var TokenSeq): int = proc getTokens(buffer: string, tokens: var TokenSeq) =
var L: Lexer var L: Lexer
var length = tokens.len var length = tokens.len
L.buf = cstring(buffer) L.buf = cstring(buffer)
@ -489,7 +489,7 @@ type
autoSymIdx: int # order of occurence: fnAutoSymbol autoSymIdx: int # order of occurence: fnAutoSymbol
label: string # valid for fnAutoNumberLabel label: string # valid for fnAutoNumberLabel
SharedState = object RstSharedState = object
options: RstParseOptions # parsing options options: RstParseOptions # parsing options
hLevels: LevelMap # hierarchy of heading styles hLevels: LevelMap # hierarchy of heading styles
hTitleCnt: int # =0 if no title, =1 if only main title, hTitleCnt: int # =0 if no title, =1 if only main title,
@ -498,8 +498,8 @@ type
currRole: string # current interpreted text role currRole: string # current interpreted text role
currRoleKind: RstNodeKind # ... and its node kind currRoleKind: RstNodeKind # ... and its node kind
subs: seq[Substitution] # substitutions subs: seq[Substitution] # substitutions
refs: seq[Substitution] # references refs*: seq[Substitution] # references
anchors: seq[AnchorSubst] # internal target substitutions anchors*: seq[AnchorSubst] # internal target substitutions
lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#]
lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_
lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*]
@ -508,19 +508,19 @@ type
# number, order of occurrence # number, order of occurrence
msgHandler: MsgHandler # How to handle errors. msgHandler: MsgHandler # How to handle errors.
findFile: FindFileHandler # How to find files. findFile: FindFileHandler # How to find files.
filename: string
hasToc*: bool
PSharedState = ref SharedState PRstSharedState* = ref RstSharedState
RstParser = object of RootObj RstParser = object of RootObj
idx*: int idx*: int
tok*: TokenSeq tok*: TokenSeq
s*: PSharedState s*: PRstSharedState
indentStack*: seq[int] indentStack*: seq[int]
filename*: string
line*, col*: int ## initial line/column of whole text or line*, col*: int ## initial line/column of whole text or
## documenation fragment that will be added ## documenation fragment that will be added
## in case of error/warning reporting to ## in case of error/warning reporting to
## (relative) line/column of the token. ## (relative) line/column of the token.
hasToc*: bool
curAnchor*: string # variable to track latest anchor in s.anchors curAnchor*: string # variable to track latest anchor in s.anchors
EParseError* = object of ValueError EParseError* = object of ValueError
@ -578,35 +578,42 @@ proc whichRoleAux(sym: string): RstNodeKind =
else: # unknown role else: # unknown role
result = rnUnknownRole result = rnUnknownRole
proc newSharedState(options: RstParseOptions, proc newRstSharedState*(options: RstParseOptions,
findFile: FindFileHandler, filename: string,
msgHandler: MsgHandler): PSharedState = findFile: FindFileHandler,
new(result) msgHandler: MsgHandler): PRstSharedState =
result.currRole = defaultRole(options) let r = defaultRole(options)
result.currRoleKind = whichRoleAux(result.currRole) result = PRstSharedState(
result.subs = @[] currRole: r,
result.refs = @[] currRoleKind: whichRoleAux(r),
result.options = options options: options,
result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
result.findFile = if not isNil(findFile): findFile else: defaultFindFile filename: filename,
findFile: if not isNil(findFile): findFile else: defaultFindFile
)
proc curLine(p: RstParser): int = p.line + currentTok(p).line proc curLine(p: RstParser): int = p.line + currentTok(p).line
proc findRelativeFile(p: RstParser; filename: string): string = proc findRelativeFile(p: RstParser; filename: string): string =
result = p.filename.splitFile.dir / filename result = p.s.filename.splitFile.dir / filename
if not fileExists(result): if not fileExists(result):
result = p.s.findFile(filename) result = p.s.findFile(filename)
proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
p.s.msgHandler(p.filename, curLine(p), p.s.msgHandler(p.s.filename, curLine(p),
p.col + currentTok(p).col, msgKind, arg) p.col + currentTok(p).col, msgKind, arg)
proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
## Print warnings for footnotes/substitutions.
## TODO: their line/column info is not known, to fix it.
s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg)
proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
p.s.msgHandler(p.filename, p.line + line, p.s.msgHandler(p.s.filename, p.line + line,
p.col + col, msgKind, arg) p.col + col, msgKind, arg)
proc rstMessage(p: RstParser, msgKind: MsgKind) = proc rstMessage(p: RstParser, msgKind: MsgKind) =
p.s.msgHandler(p.filename, curLine(p), p.s.msgHandler(p.s.filename, curLine(p),
p.col + currentTok(p).col, msgKind, p.col + currentTok(p).col, msgKind,
currentTok(p).symbol) currentTok(p).symbol)
@ -680,12 +687,10 @@ proc popInd(p: var RstParser) =
# whether it should continue its processing or not, and decided not to, # whether it should continue its processing or not, and decided not to,
# then this B.E. handler should step back (e.g. do `dec p.idx`). # then this B.E. handler should step back (e.g. do `dec p.idx`).
proc initParser(p: var RstParser, sharedState: PSharedState) = proc initParser(p: var RstParser, sharedState: PRstSharedState) =
p.indentStack = @[0] p.indentStack = @[0]
p.tok = @[] p.tok = @[]
p.idx = 0 p.idx = 0
p.filename = ""
p.hasToc = false
p.col = ColRstInit p.col = ColRstInit
p.line = LineRstInit p.line = LineRstInit
p.s = sharedState p.s = sharedState
@ -754,14 +759,14 @@ proc rstnodeToRefname(n: PRstNode): string =
var b = false var b = false
rstnodeToRefnameAux(n, result, b) rstnodeToRefnameAux(n, result, b)
proc findSub(p: var RstParser, n: PRstNode): int = proc findSub(s: PRstSharedState, n: PRstNode): int =
var key = addNodes(n) var key = addNodes(n)
# the spec says: if no exact match, try one without case distinction: # the spec says: if no exact match, try one without case distinction:
for i in countup(0, high(p.s.subs)): for i in countup(0, high(s.subs)):
if key == p.s.subs[i].key: if key == s.subs[i].key:
return i return i
for i in countup(0, high(p.s.subs)): for i in countup(0, high(s.subs)):
if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: if cmpIgnoreStyle(key, s.subs[i].key) == 0:
return i return i
result = -1 result = -1
@ -783,10 +788,10 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) =
return return
p.s.refs.add(Substitution(key: key, value: value)) p.s.refs.add(Substitution(key: key, value: value))
proc findRef(p: var RstParser, key: string): PRstNode = proc findRef(s: PRstSharedState, key: string): PRstNode =
for i in countup(0, high(p.s.refs)): for i in countup(0, high(s.refs)):
if key == p.s.refs[i].key: if key == s.refs[i].key:
return p.s.refs[i].value return s.refs[i].value
proc addAnchor(p: var RstParser, refn: string, reset: bool) = proc addAnchor(p: var RstParser, refn: string, reset: bool) =
## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor``
@ -800,8 +805,8 @@ proc addAnchor(p: var RstParser, refn: string, reset: bool) =
else: else:
p.curAnchor = refn p.curAnchor = refn
proc findMainAnchor(p: RstParser, refn: string): string = proc findMainAnchor(s: PRstSharedState, refn: string): string =
for subst in p.s.anchors: for subst in s.anchors:
if subst.mainAnchor == refn: # no need to rename if subst.mainAnchor == refn: # no need to rename
result = subst.mainAnchor result = subst.mainAnchor
break break
@ -838,28 +843,28 @@ proc addFootnoteSymAuto(p: var RstParser) =
p.s.lineFootnoteSym.add curLine(p) p.s.lineFootnoteSym.add curLine(p)
p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
proc orderFootnotes(p: var RstParser) = proc orderFootnotes(s: PRstSharedState) =
## numerate auto-numbered footnotes taking into account that all ## numerate auto-numbered footnotes taking into account that all
## manually numbered ones always have preference. ## manually numbered ones always have preference.
## Save the result back to p.s.footnotes. ## Save the result back to `s.footnotes`.
# Report an error if found any mismatch in number of automatic footnotes # Report an error if found any mismatch in number of automatic footnotes
proc listFootnotes(lines: seq[int]): string = proc listFootnotes(lines: seq[int]): string =
result.add $lines.len & " (lines " & join(lines, ", ") & ")" result.add $lines.len & " (lines " & join(lines, ", ") & ")"
if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len: if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
rstMessage(p, meFootnoteMismatch, rstMessage(s, meFootnoteMismatch,
"$1 != $2" % [listFootnotes(p.s.lineFootnoteNum), "$1 != $2" % [listFootnotes(s.lineFootnoteNum),
listFootnotes(p.s.lineFootnoteNumRef)] & listFootnotes(s.lineFootnoteNumRef)] &
" for auto-numbered footnotes") " for auto-numbered footnotes")
if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len: if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len:
rstMessage(p, meFootnoteMismatch, rstMessage(s, meFootnoteMismatch,
"$1 != $2" % [listFootnotes(p.s.lineFootnoteSym), "$1 != $2" % [listFootnotes(s.lineFootnoteSym),
listFootnotes(p.s.lineFootnoteSymRef)] & listFootnotes(s.lineFootnoteSymRef)] &
" for auto-symbol footnotes") " for auto-symbol footnotes")
var result: seq[FootnoteSubst] var result: seq[FootnoteSubst]
var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] var manuallyN, autoN, autoSymbol: seq[FootnoteSubst]
for fs in p.s.footnotes: for fs in s.footnotes:
if fs.kind == fnManualNumber: manuallyN.add fs if fs.kind == fnManualNumber: manuallyN.add fs
elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs
else: autoSymbol.add fs else: autoSymbol.add fs
@ -906,26 +911,26 @@ proc orderFootnotes(p: var RstParser) =
let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols)
result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) result.add((fs.kind, -1, -1, fs.autoSymIdx, label))
p.s.footnotes = result s.footnotes = result
proc getFootnoteNum(p: var RstParser, label: string): int = proc getFootnoteNum(s: PRstSharedState, label: string): int =
## get number from label. Must be called after `orderFootnotes`. ## get number from label. Must be called after `orderFootnotes`.
result = -1 result = -1
for fnote in p.s.footnotes: for fnote in s.footnotes:
if fnote.label == label: if fnote.label == label:
return fnote.number return fnote.number
proc getFootnoteNum(p: var RstParser, order: int): int = proc getFootnoteNum(s: PRstSharedState, order: int): int =
## get number from occurrence. Must be called after `orderFootnotes`. ## get number from occurrence. Must be called after `orderFootnotes`.
result = -1 result = -1
for fnote in p.s.footnotes: for fnote in s.footnotes:
if fnote.autoNumIdx == order: if fnote.autoNumIdx == order:
return fnote.number return fnote.number
proc getAutoSymbol(p: var RstParser, order: int): string = proc getAutoSymbol(s: PRstSharedState, order: int): string =
## get symbol from occurrence of auto-symbol footnote. ## get symbol from occurrence of auto-symbol footnote.
result = "???" result = "???"
for fnote in p.s.footnotes: for fnote in s.footnotes:
if fnote.autoSymIdx == order: if fnote.autoSymIdx == order:
return fnote.label return fnote.label
@ -1765,17 +1770,18 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
line: curLine(p), hasPeers: false) line: curLine(p), hasPeers: false)
result = p.s.hLevels.len - 1 result = p.s.hLevels.len - 1
proc countTitles(p: var RstParser, n: PRstNode) = proc countTitles(s: PRstSharedState, n: PRstNode) =
## Fill `p.s.hTitleCnt` ## Fill `s.hTitleCnt`
if n == nil: return
for node in n.sons: for node in n.sons:
if node != nil: if node != nil:
if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}:
break break
if node.kind == rnOverline: if node.kind == rnOverline:
if p.s.hLevels[p.s.hTitleCnt].hasPeers: if s.hLevels[s.hTitleCnt].hasPeers:
break break
inc p.s.hTitleCnt inc s.hTitleCnt
if p.s.hTitleCnt >= 2: if s.hTitleCnt >= 2:
break break
proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
@ -2103,8 +2109,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode =
initParser(q, p.s) initParser(q, p.s)
q.col = cols[j] q.col = cols[j]
q.line = line - 1 q.line = line - 1
q.filename = p.filename getTokens(row[j], q.tok)
q.col += getTokens(row[j], q.tok)
b = newRstNode(rnTableDataCell) b = newRstNode(rnTableDataCell)
b.add(parseDoc(q)) b.add(parseDoc(q))
a.add(b) a.add(b)
@ -2156,8 +2161,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode =
initParser(q, p.s) initParser(q, p.s)
q.col = p.col q.col = p.col
q.line = currentTok(p).line - 1 q.line = currentTok(p).line - 1
q.filename = p.filename getTokens(getColContents(p, row[j]), q.tok)
q.col += getTokens(getColContents(p, row[j]), q.tok)
b.add(parseDoc(q)) b.add(parseDoc(q))
a.add(b) a.add(b)
result.add(a) result.add(a)
@ -2562,8 +2566,8 @@ proc dirInclude(p: var RstParser): PRstNode =
var q: RstParser var q: RstParser
initParser(q, p.s) initParser(q, p.s)
q.filename = path q.s.filename = path
q.col += getTokens( getTokens(
inputString[startPosition..endPosition].strip(), inputString[startPosition..endPosition].strip(),
q.tok) q.tok)
# workaround a GCC bug; more like the interior pointer bug? # workaround a GCC bug; more like the interior pointer bug?
@ -2806,7 +2810,27 @@ proc parseDotDot(p: var RstParser): PRstNode =
else: else:
result = parseComment(p) result = parseComment(p)
proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = proc rstParsePass1*(fragment, filename: string,
line, column: int,
options: RstParseOptions,
sharedState: PRstSharedState): PRstNode =
## Parses an RST `fragment`.
## The result should be further processed by
## `preparePass2` and `resolveSubs` (which is pass 2).
var p: RstParser
initParser(p, sharedState)
p.line = line
p.col = column
getTokens(fragment, p.tok)
result = parseDoc(p)
proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) =
## Records titles in node `mainNode` and orders footnotes.
countTitles(s, mainNode)
orderFootnotes(s)
proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
## Makes pass 2 of RST parsing.
## Resolves substitutions and anchor aliases, groups footnotes. ## Resolves substitutions and anchor aliases, groups footnotes.
## Takes input node `n` and returns the same node with recursive ## Takes input node `n` and returns the same node with recursive
## substitutions in `n.sons` to `result`. ## substitutions in `n.sons` to `result`.
@ -2814,32 +2838,32 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
if n == nil: return if n == nil: return
case n.kind case n.kind
of rnSubstitutionReferences: of rnSubstitutionReferences:
var x = findSub(p, n) var x = findSub(s, n)
if x >= 0: if x >= 0:
result = p.s.subs[x].value result = s.subs[x].value
else: else:
var key = addNodes(n) var key = addNodes(n)
var e = getEnv(key) var e = getEnv(key)
if e != "": result = newLeaf(e) if e != "": result = newLeaf(e)
else: rstMessage(p, mwUnknownSubstitution, key) else: rstMessage(s, mwUnknownSubstitution, key)
of rnHeadline, rnOverline: of rnHeadline, rnOverline:
# fix up section levels depending on presence of a title and subtitle # fix up section levels depending on presence of a title and subtitle
if p.s.hTitleCnt == 2: if s.hTitleCnt == 2:
if n.level == 1: # it's the subtitle if n.level == 1: # it's the subtitle
n.level = 0 n.level = 0
elif n.level >= 2: # normal sections elif n.level >= 2: # normal sections
n.level -= 1 n.level -= 1
elif p.s.hTitleCnt == 0: elif s.hTitleCnt == 0:
n.level += 1 n.level += 1
of rnRef: of rnRef:
let refn = rstnodeToRefname(n) let refn = rstnodeToRefname(n)
var y = findRef(p, refn) var y = findRef(s, refn)
if y != nil: if y != nil:
result = newRstNode(rnHyperlink) result = newRstNode(rnHyperlink)
let text = newRstNode(rnInner, n.sons) let text = newRstNode(rnInner, n.sons)
result.sons = @[text, y] result.sons = @[text, y]
else: else:
let s = findMainAnchor(p, refn) let s = findMainAnchor(s, refn)
if s != "": if s != "":
result = newRstNode(rnInternalRef) result = newRstNode(rnInternalRef)
let text = newRstNode(rnInner, n.sons) let text = newRstNode(rnInner, n.sons)
@ -2853,16 +2877,16 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
of fnAutoNumberLabel, fnAutoNumber: of fnAutoNumberLabel, fnAutoNumber:
if fnType == fnAutoNumberLabel: if fnType == fnAutoNumberLabel:
let labelR = rstnodeToRefname(n.sons[0]) let labelR = rstnodeToRefname(n.sons[0])
num = getFootnoteNum(p, labelR) num = getFootnoteNum(s, labelR)
else: else:
num = getFootnoteNum(p, n.order) num = getFootnoteNum(s, n.order)
var nn = newRstNode(rnInner) var nn = newRstNode(rnInner)
nn.add newLeaf($num) nn.add newLeaf($num)
result.sons[0] = nn result.sons[0] = nn
of fnAutoSymbol: of fnAutoSymbol:
let sym = getAutoSymbol(p, n.order) let sym = getAutoSymbol(s, n.order)
n.sons[0].sons[0].text = sym n.sons[0].sons[0].text = sym
n.sons[1] = resolveSubs(p, n.sons[1]) n.sons[1] = resolveSubs(s, n.sons[1])
of rnFootnoteRef: of rnFootnoteRef:
var (fnType, num) = getFootnoteType(n.sons[0]) var (fnType, num) = getFootnoteType(n.sons[0])
template addLabel(number: int | string) = template addLabel(number: int | string) =
@ -2877,31 +2901,31 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
addLabel num addLabel num
refn.add $num refn.add $num
of fnAutoNumber: of fnAutoNumber:
addLabel getFootnoteNum(p, n.order) addLabel getFootnoteNum(s, n.order)
refn.add $n.order refn.add $n.order
of fnAutoNumberLabel: of fnAutoNumberLabel:
addLabel getFootnoteNum(p, rstnodeToRefname(n)) addLabel getFootnoteNum(s, rstnodeToRefname(n))
refn.add rstnodeToRefname(n) refn.add rstnodeToRefname(n)
of fnAutoSymbol: of fnAutoSymbol:
addLabel getAutoSymbol(p, n.order) addLabel getAutoSymbol(s, n.order)
refn.add $n.order refn.add $n.order
of fnCitation: of fnCitation:
result.add n.sons[0] result.add n.sons[0]
refn.add rstnodeToRefname(n) refn.add rstnodeToRefname(n)
let s = findMainAnchor(p, refn) let anch = findMainAnchor(s, refn)
if s != "": if anch != "":
result.add newLeaf(s) # add link result.add newLeaf(anch) # add link
else: else:
rstMessage(p, mwUnknownSubstitution, refn) rstMessage(s, mwUnknownSubstitution, refn)
result.add newLeaf(refn) # add link result.add newLeaf(refn) # add link
of rnLeaf: of rnLeaf:
discard discard
of rnContents: of rnContents:
p.hasToc = true s.hasToc = true
else: else:
var regroup = false var regroup = false
for i in 0 ..< n.len: for i in 0 ..< n.len:
n.sons[i] = resolveSubs(p, n.sons[i]) n.sons[i] = resolveSubs(s, n.sons[i])
if n.sons[i] != nil and n.sons[i].kind == rnFootnote: if n.sons[i] != nil and n.sons[i].kind == rnFootnote:
regroup = true regroup = true
if regroup: # group footnotes together into rnFootnoteGroup if regroup: # group footnotes together into rnFootnoteGroup
@ -2924,13 +2948,10 @@ proc rstParse*(text, filename: string,
options: RstParseOptions, options: RstParseOptions,
findFile: FindFileHandler = nil, findFile: FindFileHandler = nil,
msgHandler: MsgHandler = nil): PRstNode = msgHandler: MsgHandler = nil): PRstNode =
var p: RstParser ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`.
initParser(p, newSharedState(options, findFile, msgHandler)) var sharedState = newRstSharedState(options, filename, findFile, msgHandler)
p.filename = filename let unresolved = rstParsePass1(text, filename, line, column,
p.line = line options, sharedState)
p.col = column + getTokens(text, p.tok) preparePass2(sharedState, unresolved)
let unresolved = parseDoc(p) result = resolveSubs(sharedState, unresolved)
countTitles(p, unresolved) hasToc = sharedState.hasToc
orderFootnotes(p)
result = resolveSubs(p, unresolved)
hasToc = p.hasToc

10
nimdoc/testproject/expected/subdir/subdir_b/utils.idx
View file

@ -1,8 +1,3 @@
This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header
Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header
And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on
More headers subdir/subdir_b/utils.html#more-headers More headers
Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6
enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA
enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB
enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC
@ -11,3 +6,8 @@ someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType
aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped
bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped
fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped
This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header
Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header
And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on
More headers subdir/subdir_b/utils.html#more-headers More headers
Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6

7
tests/stdlib/trstgen.nim
View file

@ -997,7 +997,7 @@ Test1
""" """
var error5 = new string var error5 = new string
let output5 = input5.toHtml(error=error5) let output5 = input5.toHtml(error=error5)
check(error5[] == "input(6, 1) Error: mismatch in number of footnotes " & check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " &
"and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " & "and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " &
"footnotes") "footnotes")
@ -1011,7 +1011,7 @@ Test1
""" """
var error6 = new string var error6 = new string
let output6 = input6.toHtml(error=error6) let output6 = input6.toHtml(error=error6)
check(error6[] == "input(6, 1) Error: mismatch in number of footnotes " & check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " &
"and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " & "and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " &
"footnotes") "footnotes")
@ -1034,7 +1034,8 @@ Test1
""" """
var warnings8 = new seq[string] var warnings8 = new seq[string]
let output8 = input8.toHtml(warnings=warnings8) let output8 = input8.toHtml(warnings=warnings8)
check(warnings8[] == @["input(4, 1) Warning: unknown substitution " & # TODO: the line 1 is arbitrary because reference lines are not preserved
check(warnings8[] == @["input(1, 1) Warning: unknown substitution " &
"\'citation-som\'"]) "\'citation-som\'"])
# check that footnote group does not break parsing of other directives: # check that footnote group does not break parsing of other directives: