jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rst: add missing line/column info for some warnings (#18383)

Browse source 
* rst: add missing line/column info for some warnings

* add workaround

* use TLineInfo/FileIndex for storing file names

* fix blank lines in include file (rm harmful strip)

* don't use ref TLineInfo

* return `hasToc` as output parameter for uniformity

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* address review - stylistic things

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* unify RST warnings/errors names

* doAssert + minor name change

* fix a bug caught by doAssert

* apply strbasics.strip to final HTML/Latex

* rm redundant filename

* fix test after rebase

* delete `order` from rnFootnoteRef,

also display errors/warnings properly when footnote references are from
different files

* Update compiler/lineinfos.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* revert because of error:

Error: cannot prove that it's safe to initialize 'info' with the runtime value for the discriminator 'kind'

* Update lib/packages/docutils/rstgen.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* apply suggestion

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* add Table for string->file name mapping

* do not import compiler/lineinfos

* fix ambiguous calls

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>
Co-authored-by: narimiran <narimiran@disroot.org>
This commit is contained in:
Andrey Makarov 2021-07-20 09:32:22 +03:00 • committed by GitHub
commit 8c7ee96457
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 342 additions and 193 deletions
Download patch file Download diff file Expand all files Collapse all files

54
compiler/docgen.nim
View file

@ -12,11 +12,12 @@
import import
ast, strutils, strtabs, options, msgs, os, idents, ast, strutils, strtabs, options, msgs, os, idents,
wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast, wordrecg, syntaxes, renderer, lexer,
packages/docutils/rst, packages/docutils/rstgen, packages/docutils/rst, packages/docutils/rstgen,
json, xmltree, trees, types, json, xmltree, trees, types,
typesrenderer, astalgo, lineinfos, intsets, typesrenderer, astalgo, lineinfos, intsets,
pathutils, tables, nimpaths, renderverbatim, osproc pathutils, tables, nimpaths, renderverbatim, osproc
import packages/docutils/rstast except FileIndex, TLineInfo
from uri import encodeUrl from uri import encodeUrl
from std/private/globs import nativeToUnixPath from std/private/globs import nativeToUnixPath
@ -159,17 +160,18 @@ template declareClosures =
case msgKind case msgKind
of meCannotOpenFile: k = errCannotOpenFile of meCannotOpenFile: k = errCannotOpenFile
of meExpected: k = errXExpected of meExpected: k = errXExpected
of meGridTableNotImplemented: k = errGridTableNotImplemented of meGridTableNotImplemented: k = errRstGridTableNotImplemented
of meMarkdownIllformedTable: k = errMarkdownIllformedTable of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
of meNewSectionExpected: k = errNewSectionExpected of meNewSectionExpected: k = errRstNewSectionExpected
of meGeneralParseError: k = errGeneralParseError of meGeneralParseError: k = errRstGeneralParseError
of meInvalidDirective: k = errInvalidDirectiveX of meInvalidDirective: k = errRstInvalidDirectiveX
of meInvalidRstField: k = errInvalidRstField of meInvalidField: k = errRstInvalidField
of meFootnoteMismatch: k = errFootnoteMismatch of meFootnoteMismatch: k = errRstFootnoteMismatch
of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
of mwUnknownSubstitution: k = warnUnknownSubstitutionX of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
of mwUnsupportedLanguage: k = warnLanguageXNotSupported of mwBrokenLink: k = warnRstBrokenLink
of mwUnsupportedField: k = warnFieldXNotSupported of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
of mwUnsupportedField: k = warnRstFieldXNotSupported
of mwRstStyle: k = warnRstStyle of mwRstStyle: k = warnRstStyle
{.gcsafe.}: {.gcsafe.}:
globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg) globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
@ -182,11 +184,9 @@ template declareClosures =
proc parseRst(text, filename: string, proc parseRst(text, filename: string,
line, column: int, line, column: int,
rstOptions: RstParseOptions;
conf: ConfigRef, sharedState: PRstSharedState): PRstNode = conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
declareClosures() declareClosures()
result = rstParsePass1(text, filename, line, column, rstOptions, result = rstParsePass1(text, line, column, sharedState)
sharedState)
proc getOutFile2(conf: ConfigRef; filename: RelativeFile, proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
ext: string, guessTarget: bool): AbsoluteFile = ext: string, guessTarget: bool): AbsoluteFile =
@ -202,20 +202,22 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
outExt: string = HtmlExt, module: PSym = nil): PDoc = outExt: string = HtmlExt, module: PSym = nil,
isPureRst = false): PDoc =
declareClosures() declareClosures()
new(result) new(result)
result.module = module result.module = module
result.conf = conf result.conf = conf
result.cache = cache result.cache = cache
result.outDir = conf.outDir.string result.outDir = conf.outDir.string
const options = {roSupportRawDirective, roSupportMarkdown, result.isPureRst = isPureRst
roPreferMarkdown, roNimFile} var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}
if not isPureRst: options.incl roNimFile
result.sharedState = newRstSharedState( result.sharedState = newRstSharedState(
options, filename.string, options, filename.string,
docgenFindFile, compilerMsgHandler) docgenFindFile, compilerMsgHandler)
initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
conf.configVars, filename.string, options, conf.configVars, filename.string,
docgenFindFile, compilerMsgHandler) docgenFindFile, compilerMsgHandler)
if conf.configVars.hasKey("doc.googleAnalytics"): if conf.configVars.hasKey("doc.googleAnalytics"):
@ -299,8 +301,7 @@ proc genComment(d: PDoc, n: PNode): PRstNode =
result = parseRst(n.comment, toFullPath(d.conf, n.info), result = parseRst(n.comment, toFullPath(d.conf, n.info),
toLinenumber(n.info), toLinenumber(n.info),
toColumn(n.info) + DocColOffset, toColumn(n.info) + DocColOffset,
d.options, d.conf, d.conf, d.sharedState)
d.sharedState)
proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
if n == nil: return nil if n == nil: return nil
@ -1123,6 +1124,9 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
proc finishGenerateDoc*(d: var PDoc) = proc finishGenerateDoc*(d: var PDoc) =
## Perform 2nd RST pass for resolution of links/footnotes/headings... ## Perform 2nd RST pass for resolution of links/footnotes/headings...
# copy file map `filenames` to ``rstgen.nim`` for its warnings
d.filenames = d.sharedState.filenames
# Main title/subtitle are allowed only in the first RST fragment of document # Main title/subtitle are allowed only in the first RST fragment of document
var firstRst = PRstNode(nil) var firstRst = PRstNode(nil)
for fragment in d.modDescPre: for fragment in d.modDescPre:
@ -1417,14 +1421,10 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
proc commandRstAux(cache: IdentCache, conf: ConfigRef; proc commandRstAux(cache: IdentCache, conf: ConfigRef;
filename: AbsoluteFile, outExt: string) = filename: AbsoluteFile, outExt: string) =
var filen = addFileExt(filename, "txt") var filen = addFileExt(filename, "txt")
var d = newDocumentor(filen, cache, conf, outExt) var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
d.isPureRst = true
let rst = parseRst(readFile(filen.string), filen.string, let rst = parseRst(readFile(filen.string), filen.string,
line=LineRstInit, column=ColRstInit, line=LineRstInit, column=ColRstInit,
{roSupportRawDirective, roSupportMarkdown, conf, d.sharedState)
roPreferMarkdown}, conf,
d.sharedState)
d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
finishGenerateDoc(d) finishGenerateDoc(d)
writeOutput(d) writeOutput(d)

49
compiler/lineinfos.nim
View file

@ -32,13 +32,13 @@ type
# non-fatal errors # non-fatal errors
errIllFormedAstX, errCannotOpenFile, errIllFormedAstX, errCannotOpenFile,
errXExpected, errXExpected,
errGridTableNotImplemented, errRstGridTableNotImplemented,
errMarkdownIllformedTable, errRstMarkdownIllformedTable,
errGeneralParseError, errRstNewSectionExpected,
errNewSectionExpected, errRstGeneralParseError,
errInvalidDirectiveX, errRstInvalidDirectiveX,
errInvalidRstField, errRstInvalidField,
errFootnoteMismatch, errRstFootnoteMismatch,
errProveInit, # deadcode errProveInit, # deadcode
errGenerated, errGenerated,
errUser, errUser,
@ -47,10 +47,13 @@ type
warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX", warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
warnLanguageXNotSupported = "LanguageXNotSupported", warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
warnFieldXNotSupported = "FieldXNotSupported", warnRstBrokenLink = "BrokenLink",
warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored", warnRstLanguageXNotSupported = "LanguageXNotSupported",
warnRstFieldXNotSupported = "FieldXNotSupported",
warnRstStyle = "warnRstStyle",
warnCommentXIgnored = "CommentXIgnored",
warnTypelessParam = "TypelessParam", warnTypelessParam = "TypelessParam",
warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap", warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport", warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
@ -93,13 +96,13 @@ const
errIllFormedAstX: "illformed AST: $1", errIllFormedAstX: "illformed AST: $1",
errCannotOpenFile: "cannot open '$1'", errCannotOpenFile: "cannot open '$1'",
errXExpected: "'$1' expected", errXExpected: "'$1' expected",
errGridTableNotImplemented: "grid table is not implemented", errRstGridTableNotImplemented: "grid table is not implemented",
errMarkdownIllformedTable: "illformed delimiter row of a markdown table", errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
errGeneralParseError: "general parse error", errRstNewSectionExpected: "new section expected $1",
errNewSectionExpected: "new section expected $1", errRstGeneralParseError: "general parse error",
errInvalidDirectiveX: "invalid directive: '$1'", errRstInvalidDirectiveX: "invalid directive: '$1'",
errInvalidRstField: "invalid field: $1", errRstInvalidField: "invalid field: $1",
errFootnoteMismatch: "number of footnotes and their references don't match: $1", errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errProveInit: "Cannot prove that '$1' is initialized.", # deadcode
errGenerated: "$1", errGenerated: "$1",
errUser: "$1", errUser: "$1",
@ -111,10 +114,11 @@ const
warnConfigDeprecated: "config file '$1' is deprecated", warnConfigDeprecated: "config file '$1' is deprecated",
warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)", warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnUnknownMagic: "unknown magic '$1' might crash the compiler",
warnRedefinitionOfLabel: "redefinition of label '$1'", warnRstRedefinitionOfLabel: "redefinition of label '$1'",
warnUnknownSubstitutionX: "unknown substitution '$1'", warnRstUnknownSubstitutionX: "unknown substitution '$1'",
warnLanguageXNotSupported: "language '$1' not supported", warnRstBrokenLink: "broken link '$1'",
warnFieldXNotSupported: "field '$1' not supported", warnRstLanguageXNotSupported: "language '$1' not supported",
warnRstFieldXNotSupported: "field '$1' not supported",
warnRstStyle: "RST style: $1", warnRstStyle: "RST style: $1",
warnCommentXIgnored: "comment '$1' ignored", warnCommentXIgnored: "comment '$1' ignored",
warnTypelessParam: "", # deadcode warnTypelessParam: "", # deadcode
@ -196,6 +200,7 @@ const
warnMax* = pred(hintSuccess) warnMax* = pred(hintSuccess)
hintMin* = hintSuccess hintMin* = hintSuccess
hintMax* = high(TMsgKind) hintMax* = high(TMsgKind)
rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle}
type type
TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints

11
compiler/main.nim
View file

@ -285,7 +285,7 @@ proc mainCommand*(graph: ModuleGraph) =
of cmdDoc: of cmdDoc:
docLikeCmd(): docLikeCmd():
conf.setNoteDefaults(warnLockLevel, false) # issue #13218 conf.setNoteDefaults(warnLockLevel, false) # issue #13218
conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218
# because currently generates lots of false positives due to conflation # because currently generates lots of false positives due to conflation
# of labels links in doc comments, e.g. for random.rand: # of labels links in doc comments, e.g. for random.rand:
# ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer # ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
@ -295,19 +295,16 @@ proc mainCommand*(graph: ModuleGraph) =
commandBuildIndex(conf, $conf.outDir) commandBuildIndex(conf, $conf.outDir)
of cmdRst2html: of cmdRst2html:
# XXX: why are warnings disabled by default for rst2html and rst2tex? # XXX: why are warnings disabled by default for rst2html and rst2tex?
for warn in [warnUnknownSubstitutionX, warnLanguageXNotSupported, for warn in rstWarnings:
warnFieldXNotSupported, warnRstStyle]:
conf.setNoteDefaults(warn, true) conf.setNoteDefaults(warn, true)
conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218 conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218
when defined(leanCompiler): when defined(leanCompiler):
conf.quitOrRaise "compiler wasn't built with documentation generator" conf.quitOrRaise "compiler wasn't built with documentation generator"
else: else:
loadConfigs(DocConfig, cache, conf, graph.idgen) loadConfigs(DocConfig, cache, conf, graph.idgen)
commandRst2Html(cache, conf) commandRst2Html(cache, conf)
of cmdRst2tex, cmdDoc2tex: of cmdRst2tex, cmdDoc2tex:
for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX, for warn in rstWarnings:
warnLanguageXNotSupported,
warnFieldXNotSupported, warnRstStyle]:
conf.setNoteDefaults(warn, true) conf.setNoteDefaults(warn, true)
when defined(leanCompiler): when defined(leanCompiler):
conf.quitOrRaise "compiler wasn't built with documentation generator" conf.quitOrRaise "compiler wasn't built with documentation generator"

179
lib/packages/docutils/rst.nim
View file

@ -196,7 +196,7 @@
import import
os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
std/private/miscdollars std/private/miscdollars, tables
from highlite import SourceLanguage, getSourceLanguage from highlite import SourceLanguage, getSourceLanguage
type type
@ -217,6 +217,7 @@ type
mcWarning = "Warning", mcWarning = "Warning",
mcError = "Error" mcError = "Error"
# keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim:
MsgKind* = enum ## the possible messages MsgKind* = enum ## the possible messages
meCannotOpenFile = "cannot open '$1'", meCannotOpenFile = "cannot open '$1'",
meExpected = "'$1' expected", meExpected = "'$1' expected",
@ -225,10 +226,11 @@ type
meNewSectionExpected = "new section expected $1", meNewSectionExpected = "new section expected $1",
meGeneralParseError = "general parse error", meGeneralParseError = "general parse error",
meInvalidDirective = "invalid directive: '$1'", meInvalidDirective = "invalid directive: '$1'",
meInvalidRstField = "invalid field: $1", meInvalidField = "invalid field: $1",
meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1",
mwRedefinitionOfLabel = "redefinition of label '$1'", mwRedefinitionOfLabel = "redefinition of label '$1'",
mwUnknownSubstitution = "unknown substitution '$1'", mwUnknownSubstitution = "unknown substitution '$1'",
mwBrokenLink = "broken link '$1'",
mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedLanguage = "language '$1' not supported",
mwUnsupportedField = "field '$1' not supported", mwUnsupportedField = "field '$1' not supported",
mwRstStyle = "RST style: $1" mwRstStyle = "RST style: $1"
@ -489,7 +491,9 @@ type
autoNumIdx: int # order of occurence: fnAutoNumber, fnAutoNumberLabel autoNumIdx: int # order of occurence: fnAutoNumber, fnAutoNumberLabel
autoSymIdx: int # order of occurence: fnAutoSymbol autoSymIdx: int # order of occurence: fnAutoSymbol
label: string # valid for fnAutoNumberLabel label: string # valid for fnAutoNumberLabel
RstFileTable* = object
filenameToIdx*: Table[string, FileIndex]
idxToFilename*: seq[string]
RstSharedState = object RstSharedState = object
options: RstParseOptions # parsing options options: RstParseOptions # parsing options
hLevels: LevelMap # hierarchy of heading styles hLevels: LevelMap # hierarchy of heading styles
@ -501,15 +505,19 @@ type
subs: seq[Substitution] # substitutions subs: seq[Substitution] # substitutions
refs*: seq[Substitution] # references refs*: seq[Substitution] # references
anchors*: seq[AnchorSubst] # internal target substitutions anchors*: seq[AnchorSubst] # internal target substitutions
lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#]
lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_
lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] currFootnoteNumRef: int # ... their counter for `resolveSubs`
lineFootnoteSymRef: seq[int] # footnote line, their reference [*]_ lineFootnoteSym: seq[TLineInfo] # footnote line, auto symbols .. [*]
lineFootnoteSymRef: seq[TLineInfo] # footnote line, their reference [*]_
currFootnoteSymRef: int # ... their counter for `resolveSubs`
footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
# number, order of occurrence # number, order of occurrence
msgHandler: MsgHandler # How to handle errors. msgHandler: MsgHandler # How to handle errors.
findFile: FindFileHandler # How to find files. findFile: FindFileHandler # How to find files.
filename: string filenames*: RstFileTable # map file name <-> FileIndex (for storing
# file names for warnings after 1st stage)
currFileIdx: FileIndex # current index in `filesnames`
hasToc*: bool hasToc*: bool
PRstSharedState* = ref RstSharedState PRstSharedState* = ref RstSharedState
@ -579,6 +587,25 @@ proc whichRoleAux(sym: string): RstNodeKind =
else: # unknown role else: # unknown role
result = rnUnknownRole result = rnUnknownRole
proc len(filenames: RstFileTable): int = filenames.idxToFilename.len
proc setCurrFilename(s: PRstSharedState, file1: string) =
let nextIdx = s.filenames.len.FileIndex
let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
if v == nextIdx:
s.filenames.filenameToIdx[file1] = v
s.filenames.idxToFilename.add file1
s.currFileIdx = v
proc getFilename(filenames: RstFileTable, fid: FileIndex): string =
doAssert(0 <= fid.int and fid.int < filenames.len,
"incorrect FileIndex $1 (range 0..$2)" % [
$fid.int, $(filenames.len - 1)])
result = filenames.idxToFilename[fid.int]
proc currFilename(s: PRstSharedState): string =
getFilename(s.filenames, s.currFileIdx)
proc newRstSharedState*(options: RstParseOptions, proc newRstSharedState*(options: RstParseOptions,
filename: string, filename: string,
findFile: FindFileHandler, findFile: FindFileHandler,
@ -589,32 +616,37 @@ proc newRstSharedState*(options: RstParseOptions,
currRoleKind: whichRoleAux(r), currRoleKind: whichRoleAux(r),
options: options, options: options,
msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
filename: filename,
findFile: if not isNil(findFile): findFile else: defaultFindFile findFile: if not isNil(findFile): findFile else: defaultFindFile
) )
setCurrFilename(result, filename)
proc curLine(p: RstParser): int = p.line + currentTok(p).line proc curLine(p: RstParser): int = p.line + currentTok(p).line
proc findRelativeFile(p: RstParser; filename: string): string = proc findRelativeFile(p: RstParser; filename: string): string =
result = p.s.filename.splitFile.dir / filename result = p.s.currFilename.splitFile.dir / filename
if not fileExists(result): if not fileExists(result):
result = p.s.findFile(filename) result = p.s.findFile(filename)
proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
p.s.msgHandler(p.s.filename, curLine(p), p.s.msgHandler(p.s.currFilename, curLine(p),
p.col + currentTok(p).col, msgKind, arg) p.col + currentTok(p).col, msgKind, arg)
proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
## Print warnings for footnotes/substitutions. s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg)
## TODO: their line/column info is not known, to fix it.
s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) proc rstMessage*(filenames: RstFileTable, f: MsgHandler,
info: TLineInfo, msgKind: MsgKind, arg: string) =
## Print warnings using `info`, i.e. in 2nd-pass warnings for
## footnotes/substitutions/references or from ``rstgen.nim``.
let file = getFilename(filenames, info.fileIndex)
f(file, info.line.int, info.col.int, msgKind, arg)
proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
p.s.msgHandler(p.s.filename, p.line + line, p.s.msgHandler(p.s.currFilename, p.line + line,
p.col + col, msgKind, arg) p.col + col, msgKind, arg)
proc rstMessage(p: RstParser, msgKind: MsgKind) = proc rstMessage(p: RstParser, msgKind: MsgKind) =
p.s.msgHandler(p.s.filename, curLine(p), p.s.msgHandler(p.s.currFilename, curLine(p),
p.col + currentTok(p).col, msgKind, p.col + currentTok(p).col, msgKind,
currentTok(p).symbol) currentTok(p).symbol)
@ -827,11 +859,18 @@ proc addFootnoteNumManual(p: var RstParser, num: int) =
return return
p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) p.s.footnotes.add((fnManualNumber, num, -1, -1, $num))
proc lineInfo(p: RstParser, iTok: int): TLineInfo =
result.col = int16(p.col + p.tok[iTok].col)
result.line = uint16(p.line + p.tok[iTok].line)
result.fileIndex = p.s.currFileIdx
proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
proc addFootnoteNumAuto(p: var RstParser, label: string) = proc addFootnoteNumAuto(p: var RstParser, label: string) =
## add auto-numbered footnote. ## add auto-numbered footnote.
## Empty label [#] means it'll be resolved by the occurrence. ## Empty label [#] means it'll be resolved by the occurrence.
if label == "": # simple auto-numbered [#] if label == "": # simple auto-numbered [#]
p.s.lineFootnoteNum.add curLine(p) p.s.lineFootnoteNum.add lineInfo(p)
p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label)) p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label))
else: # auto-numbered with label [#label] else: # auto-numbered with label [#label]
for fnote in p.s.footnotes: for fnote in p.s.footnotes:
@ -841,7 +880,7 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) =
p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label)) p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label))
proc addFootnoteSymAuto(p: var RstParser) = proc addFootnoteSymAuto(p: var RstParser) =
p.s.lineFootnoteSym.add curLine(p) p.s.lineFootnoteSym.add lineInfo(p)
p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
proc orderFootnotes(s: PRstSharedState) = proc orderFootnotes(s: PRstSharedState) =
@ -850,7 +889,15 @@ proc orderFootnotes(s: PRstSharedState) =
## Save the result back to `s.footnotes`. ## Save the result back to `s.footnotes`.
# Report an error if found any mismatch in number of automatic footnotes # Report an error if found any mismatch in number of automatic footnotes
proc listFootnotes(lines: seq[int]): string = proc listFootnotes(locations: seq[TLineInfo]): string =
var lines: seq[string]
for info in locations:
if s.filenames.len > 1:
let file = getFilename(s.filenames, info.fileIndex)
lines.add file & ":"
else: # no need to add file name here if there is only 1
lines.add ""
lines[^1].add $info.line
result.add $lines.len & " (lines " & join(lines, ", ") & ")" result.add $lines.len & " (lines " & join(lines, ", ") & ")"
if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
rstMessage(s, meFootnoteMismatch, rstMessage(s, meFootnoteMismatch,
@ -1157,7 +1204,7 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind =
proc toInlineCode(n: PRstNode, language: string): PRstNode = proc toInlineCode(n: PRstNode, language: string): PRstNode =
## Creates rnInlineCode and attaches `n` contents as code (in 3rd son). ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
result = newRstNode(rnInlineCode) result = newRstNode(rnInlineCode, info=n.info)
let args = newRstNode(rnDirArg) let args = newRstNode(rnDirArg)
var lang = language var lang = language
if language == "cpp": lang = "c++" if language == "cpp": lang = "c++"
@ -1179,6 +1226,7 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
result = newRstNode(kind, newSons) result = newRstNode(kind, newSons)
proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
## Finalizes node `n` that was tentatively determined as interpreted text.
var newKind = n.kind var newKind = n.kind
var newSons = n.sons var newSons = n.sons
@ -1207,12 +1255,10 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
newKind = rnHyperlink newKind = rnHyperlink
newSons = @[a, b] newSons = @[a, b]
setRef(p, rstnodeToRefname(a), b) setRef(p, rstnodeToRefname(a), b)
elif n.kind == rnInterpretedText:
newKind = rnRef
else:
newKind = rnRef
newSons = @[n]
result = newRstNode(newKind, newSons) result = newRstNode(newKind, newSons)
else: # some link that will be resolved in `resolveSubs`
newKind = rnRef
result = newRstNode(newKind, sons=newSons, info=n.info)
elif match(p, p.idx, ":w:"): elif match(p, p.idx, ":w:"):
# a role: # a role:
let (roleName, lastIdx) = getRefname(p, p.idx+1) let (roleName, lastIdx) = getRefname(p, p.idx+1)
@ -1300,20 +1346,19 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
else: else:
# check for reference (probably, long one like some.ref.with.dots_ ) # check for reference (probably, long one like some.ref.with.dots_ )
var saveIdx = p.idx var saveIdx = p.idx
var isRef = false var reference: PRstNode = nil
inc p.idx inc p.idx
while currentTok(p).kind in {tkWord, tkPunct}: while currentTok(p).kind in {tkWord, tkPunct}:
if currentTok(p).kind == tkPunct: if currentTok(p).kind == tkPunct:
if isInlineMarkupEnd(p, "_", exact=true): if isInlineMarkupEnd(p, "_", exact=true):
isRef = true reference = newRstNode(rnRef, info=lineInfo(p, saveIdx))
break break
if not validRefnamePunct(currentTok(p).symbol): if not validRefnamePunct(currentTok(p).symbol):
break break
inc p.idx inc p.idx
if isRef: if reference != nil:
let r = newRstNode(rnRef) for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol)
for i in saveIdx..p.idx-1: r.add newLeaf(p.tok[i].symbol) father.add reference
father.add r
inc p.idx # skip final _ inc p.idx # skip final _
else: # 1 normal word else: # 1 normal word
father.add newLeaf(p.tok[saveIdx].symbol) father.add newLeaf(p.tok[saveIdx].symbol)
@ -1387,6 +1432,8 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
else: rstMessage(p, meExpected, postfix, line, col) else: rstMessage(p, meExpected, postfix, line, col)
proc parseMarkdownCodeblock(p: var RstParser): PRstNode = proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
result = newRstNodeA(p, rnCodeBlock)
result.info = lineInfo(p)
var args = newRstNode(rnDirArg) var args = newRstNode(rnDirArg)
if currentTok(p).kind == tkWord: if currentTok(p).kind == tkWord:
args.add(newLeaf(p)) args.add(newLeaf(p))
@ -1411,7 +1458,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
inc p.idx inc p.idx
var lb = newRstNode(rnLiteralBlock) var lb = newRstNode(rnLiteralBlock)
lb.add(n) lb.add(n)
result = newRstNodeA(p, rnCodeBlock)
result.add(args) result.add(args)
result.add(PRstNode(nil)) result.add(PRstNode(nil))
result.add(lb) result.add(lb)
@ -1495,6 +1541,7 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
proc parseInline(p: var RstParser, father: PRstNode) = proc parseInline(p: var RstParser, father: PRstNode) =
var n: PRstNode # to be used in `if` condition var n: PRstNode # to be used in `if` condition
let saveIdx = p.idx
case currentTok(p).kind case currentTok(p).kind
of tkPunct: of tkPunct:
if isInlineMarkupStart(p, "***"): if isInlineMarkupStart(p, "***"):
@ -1537,12 +1584,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
n = n.toOtherRole(k, roleName) n = n.toOtherRole(k, roleName)
father.add(n) father.add(n)
elif isInlineMarkupStart(p, "`"): elif isInlineMarkupStart(p, "`"):
var n = newRstNode(rnInterpretedText) var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1))
parseUntil(p, n, "`", false) # bug #17260 parseUntil(p, n, "`", false) # bug #17260
n = parsePostfix(p, n) n = parsePostfix(p, n)
father.add(n) father.add(n)
elif isInlineMarkupStart(p, "|"): elif isInlineMarkupStart(p, "|"):
var n = newRstNode(rnSubstitutionReferences) var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1))
parseUntil(p, n, "|", false) parseUntil(p, n, "|", false)
father.add(n) father.add(n)
elif roSupportMarkdown in p.s.options and elif roSupportMarkdown in p.s.options and
@ -1552,15 +1599,14 @@ proc parseInline(p: var RstParser, father: PRstNode) =
elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and
(n = parseFootnoteName(p, reference=true); n != nil): (n = parseFootnoteName(p, reference=true); n != nil):
var nn = newRstNode(rnFootnoteRef) var nn = newRstNode(rnFootnoteRef)
nn.info = lineInfo(p, saveIdx+1)
nn.add n nn.add n
let (fnType, _) = getFootnoteType(n) let (fnType, _) = getFootnoteType(n)
case fnType case fnType
of fnAutoSymbol: of fnAutoSymbol:
p.s.lineFootnoteSymRef.add curLine(p) p.s.lineFootnoteSymRef.add lineInfo(p)
nn.order = p.s.lineFootnoteSymRef.len
of fnAutoNumber: of fnAutoNumber:
p.s.lineFootnoteNumRef.add curLine(p) p.s.lineFootnoteNumRef.add lineInfo(p)
nn.order = p.s.lineFootnoteNumRef.len
else: discard else: discard
father.add(nn) father.add(nn)
else: else:
@ -1693,7 +1739,7 @@ proc parseField(p: var RstParser): PRstNode =
## Returns a parsed rnField node. ## Returns a parsed rnField node.
## ##
## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody. ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody.
result = newRstNode(rnField) result = newRstNode(rnField, info=lineInfo(p))
var col = currentTok(p).col var col = currentTok(p).col
var fieldname = newRstNode(rnFieldName) var fieldname = newRstNode(rnFieldName)
parseUntil(p, fieldname, ":", false) parseUntil(p, fieldname, ":", false)
@ -2469,6 +2515,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
## Both rnDirArg and rnFieldList children nodes might be nil, so you need to ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to
## check them before accessing. ## check them before accessing.
result = newRstNodeA(p, k) result = newRstNodeA(p, k)
if k == rnCodeBlock: result.info = lineInfo(p)
var args: PRstNode = nil var args: PRstNode = nil
var options: PRstNode = nil var options: PRstNode = nil
if hasArg in flags: if hasArg in flags:
@ -2593,14 +2640,16 @@ proc dirInclude(p: var RstParser): PRstNode =
var q: RstParser var q: RstParser
initParser(q, p.s) initParser(q, p.s)
q.s.filename = path let saveFileIdx = p.s.currFileIdx
setCurrFilename(p.s, path)
getTokens( getTokens(
inputString[startPosition..endPosition].strip(), inputString[startPosition..endPosition],
q.tok) q.tok)
# workaround a GCC bug; more like the interior pointer bug? # workaround a GCC bug; more like the interior pointer bug?
#if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0:
# InternalError("Too many binary zeros in include file") # InternalError("Too many binary zeros in include file")
result = parseDoc(q) result = parseDoc(q)
p.s.currFileIdx = saveFileIdx
proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
## Parses a code block. ## Parses a code block.
@ -2634,7 +2683,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList) if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
assert result.sons[1].kind == rnFieldList assert result.sons[1].kind == rnFieldList
# Hook the extra field and specify the Nim language as value. # Hook the extra field and specify the Nim language as value.
var extraNode = newRstNode(rnField) var extraNode = newRstNode(rnField, info=lineInfo(p))
extraNode.add(newRstNode(rnFieldName)) extraNode.add(newRstNode(rnFieldName))
extraNode.add(newRstNode(rnFieldBody)) extraNode.add(newRstNode(rnFieldBody))
extraNode.sons[0].add newLeaf("default-language") extraNode.sons[0].add newLeaf("default-language")
@ -2837,9 +2886,8 @@ proc parseDotDot(p: var RstParser): PRstNode =
else: else:
result = parseComment(p, col) result = parseComment(p, col)
proc rstParsePass1*(fragment, filename: string, proc rstParsePass1*(fragment: string,
line, column: int, line, column: int,
options: RstParseOptions,
sharedState: PRstSharedState): PRstNode = sharedState: PRstSharedState): PRstNode =
## Parses an RST `fragment`. ## Parses an RST `fragment`.
## The result should be further processed by ## The result should be further processed by
@ -2872,7 +2920,8 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
var key = addNodes(n) var key = addNodes(n)
var e = getEnv(key) var e = getEnv(key)
if e != "": result = newLeaf(e) if e != "": result = newLeaf(e)
else: rstMessage(s, mwUnknownSubstitution, key) else: rstMessage(s.filenames, s.msgHandler, n.info,
mwUnknownSubstitution, key)
of rnHeadline, rnOverline: of rnHeadline, rnOverline:
# fix up section levels depending on presence of a title and subtitle # fix up section levels depending on presence of a title and subtitle
if s.hTitleCnt == 2: if s.hTitleCnt == 2:
@ -2890,12 +2939,14 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
let text = newRstNode(rnInner, n.sons) let text = newRstNode(rnInner, n.sons)
result.sons = @[text, y] result.sons = @[text, y]
else: else:
let s = findMainAnchor(s, refn) let anchor = findMainAnchor(s, refn)
if s != "": if anchor != "":
result = newRstNode(rnInternalRef) result = newRstNode(rnInternalRef)
let text = newRstNode(rnInner, n.sons) let text = newRstNode(rnInner, n.sons)
result.sons = @[text, # visible text of reference result.sons = @[text, # visible text of reference
newLeaf(s)] # link itself newLeaf(anchor)] # link itself
else:
rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
of rnFootnote: of rnFootnote:
var (fnType, num) = getFootnoteType(n.sons[0]) var (fnType, num) = getFootnoteType(n.sons[0])
case fnType case fnType
@ -2922,20 +2973,22 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
result.add(nn) result.add(nn)
var refn = fnType.prefix var refn = fnType.prefix
# create new rnFootnoteRef, add final label, and finalize target refn: # create new rnFootnoteRef, add final label, and finalize target refn:
result = newRstNode(rnFootnoteRef) result = newRstNode(rnFootnoteRef, info = n.info)
case fnType case fnType
of fnManualNumber: of fnManualNumber:
addLabel num addLabel num
refn.add $num refn.add $num
of fnAutoNumber: of fnAutoNumber:
addLabel getFootnoteNum(s, n.order) inc s.currFootnoteNumRef
refn.add $n.order addLabel getFootnoteNum(s, s.currFootnoteNumRef)
refn.add $s.currFootnoteNumRef
of fnAutoNumberLabel: of fnAutoNumberLabel:
addLabel getFootnoteNum(s, rstnodeToRefname(n)) addLabel getFootnoteNum(s, rstnodeToRefname(n))
refn.add rstnodeToRefname(n) refn.add rstnodeToRefname(n)
of fnAutoSymbol: of fnAutoSymbol:
addLabel getAutoSymbol(s, n.order) inc s.currFootnoteSymRef
refn.add $n.order addLabel getAutoSymbol(s, s.currFootnoteSymRef)
refn.add $s.currFootnoteSymRef
of fnCitation: of fnCitation:
result.add n.sons[0] result.add n.sons[0]
refn.add rstnodeToRefname(n) refn.add rstnodeToRefname(n)
@ -2943,7 +2996,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
if anch != "": if anch != "":
result.add newLeaf(anch) # add link result.add newLeaf(anch) # add link
else: else:
rstMessage(s, mwUnknownSubstitution, refn) rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
result.add newLeaf(refn) # add link result.add newLeaf(refn) # add link
of rnLeaf: of rnLeaf:
discard discard
@ -2971,14 +3024,18 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
result.sons = newSons result.sons = newSons
proc rstParse*(text, filename: string, proc rstParse*(text, filename: string,
line, column: int, hasToc: var bool, line, column: int,
options: RstParseOptions, options: RstParseOptions,
findFile: FindFileHandler = nil, findFile: FindFileHandler = nil,
msgHandler: MsgHandler = nil): PRstNode = msgHandler: MsgHandler = nil):
## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`. tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] =
## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`,
## note that 2nd tuple element should be fed to `initRstGenerator`
## argument `filenames` (it is being filled here at least with `filename`
## and possibly with other files from RST ``.. include::`` statement).
var sharedState = newRstSharedState(options, filename, findFile, msgHandler) var sharedState = newRstSharedState(options, filename, findFile, msgHandler)
let unresolved = rstParsePass1(text, filename, line, column, let unresolved = rstParsePass1(text, line, column, sharedState)
options, sharedState)
preparePass2(sharedState, unresolved) preparePass2(sharedState, unresolved)
result = resolveSubs(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved)
hasToc = sharedState.hasToc result.filenames = sharedState.filenames
result.hasToc = sharedState.hasToc

20
lib/packages/docutils/rstast.nim
View file

@ -74,6 +74,11 @@ type
rnLeaf # a leaf; the node's text field contains the rnLeaf # a leaf; the node's text field contains the
# leaf val # leaf val
FileIndex* = distinct int32
TLineInfo* = object
line*: uint16
col*: int16
fileIndex*: FileIndex
PRstNode* = ref RstNode ## an RST node PRstNode* = ref RstNode ## an RST node
RstNodeSeq* = seq[PRstNode] RstNodeSeq* = seq[PRstNode]
@ -92,21 +97,32 @@ type
level*: int ## level of headings starting from 1 (main level*: int ## level of headings starting from 1 (main
## chapter) to larger ones (minor sub-sections) ## chapter) to larger ones (minor sub-sections)
## level=0 means it's document title or subtitle ## level=0 means it's document title or subtitle
of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: of rnFootnote, rnCitation, rnOptionListItem:
order*: int ## footnote order (for auto-symbol footnotes and order*: int ## footnote order (for auto-symbol footnotes and
## auto-numbered ones without a label) ## auto-numbered ones without a label)
of rnRef, rnSubstitutionReferences,
rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef:
info*: TLineInfo ## To have line/column info for warnings at
## nodes that are post-processed after parsing
else: else:
discard discard
anchor*: string ## anchor, internal link target anchor*: string ## anchor, internal link target
## (aka HTML id tag, aka Latex label/hypertarget) ## (aka HTML id tag, aka Latex label/hypertarget)
sons*: RstNodeSeq ## the node's sons sons*: RstNodeSeq ## the node's sons
proc `==`*(a, b: FileIndex): bool {.borrow.}
proc len*(n: PRstNode): int = proc len*(n: PRstNode): int =
result = len(n.sons) result = len(n.sons)
proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[], proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[],
anchor = ""): PRstNode = anchor = ""): PRstNode =
result = PRstNode(kind: kind, sons: sons, anchor: anchor)
proc newRstNode*(kind: RstNodeKind, info: TLineInfo,
sons: seq[PRstNode] = @[]): PRstNode =
result = PRstNode(kind: kind, sons: sons) result = PRstNode(kind: kind, sons: sons)
result.info = info
proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} = proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} =
assert kind in {rnLeaf, rnSmiley} assert kind in {rnLeaf, rnSmiley}
@ -388,7 +404,7 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
result.add " adType=" & node.adType result.add " adType=" & node.adType
of rnHeadline, rnOverline, rnMarkdownHeadline: of rnHeadline, rnOverline, rnMarkdownHeadline:
result.add " level=" & $node.level result.add " level=" & $node.level
of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem: of rnFootnote, rnCitation, rnOptionListItem:
result.add (if node.order == 0: "" else: " order=" & $node.order) result.add (if node.order == 0: "" else: " order=" & $node.order)
else: else:
discard discard

40
lib/packages/docutils/rstgen.nim
View file

@ -40,7 +40,7 @@
## can be done by simply searching for [footnoteName]. ## can be done by simply searching for [footnoteName].
import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
algorithm, parseutils algorithm, parseutils, std/strbasics
import ../../std/private/since import ../../std/private/since
@ -72,11 +72,11 @@ type
tocPart*: seq[TocEntry] tocPart*: seq[TocEntry]
hasToc*: bool hasToc*: bool
theIndex: string # Contents of the index file to be dumped at the end. theIndex: string # Contents of the index file to be dumped at the end.
options*: RstParseOptions
findFile*: FindFileHandler findFile*: FindFileHandler
msgHandler*: MsgHandler msgHandler*: MsgHandler
outDir*: string ## output directory, initialized by docgen.nim outDir*: string ## output directory, initialized by docgen.nim
destFile*: string ## output (HTML) file, initialized by docgen.nim destFile*: string ## output (HTML) file, initialized by docgen.nim
filenames*: RstFileTable
filename*: string ## source Nim or Rst file filename*: string ## source Nim or Rst file
meta*: array[MetaEnum, string] meta*: array[MetaEnum, string]
currentSection: string ## \ currentSection: string ## \
@ -112,9 +112,9 @@ proc init(p: var CodeBlockParams) =
proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
config: StringTableRef, filename: string, config: StringTableRef, filename: string,
options: RstParseOptions,
findFile: FindFileHandler = nil, findFile: FindFileHandler = nil,
msgHandler: MsgHandler = nil) = msgHandler: MsgHandler = nil,
filenames = default(RstFileTable)) =
## Initializes a ``RstGenerator``. ## Initializes a ``RstGenerator``.
## ##
## You need to call this before using a ``RstGenerator`` with any other ## You need to call this before using a ``RstGenerator`` with any other
@ -160,9 +160,9 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
g.target = target g.target = target
g.tocPart = @[] g.tocPart = @[]
g.filename = filename g.filename = filename
g.filenames = filenames
g.splitAfter = 20 g.splitAfter = 20
g.theIndex = "" g.theIndex = ""
g.options = options
g.findFile = findFile g.findFile = findFile
g.currentSection = "" g.currentSection = ""
g.id = 0 g.id = 0
@ -908,9 +908,8 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
[d.config.getOrDefault"doc.smiley_format" % n.text]) [d.config.getOrDefault"doc.smiley_format" % n.text])
proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int =
# TODO: proper column/line info
template err(msg: string) = template err(msg: string) =
d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg) rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg)
let value = n.getFieldValue let value = n.getFieldValue
var number: int var number: int
let nChars = parseInt(value, number) let nChars = parseInt(value, number)
@ -958,7 +957,8 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
params.langStr = n.getFieldValue.strip params.langStr = n.getFieldValue.strip
params.lang = params.langStr.getSourceLanguage params.lang = params.langStr.getSourceLanguage
else: else:
d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument) rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField,
n.getArgument)
proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
## Iterates over all code block fields and returns processed params. ## Iterates over all code block fields and returns processed params.
@ -1069,7 +1069,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
dispA(d.target, result, blockStart, blockStart, []) dispA(d.target, result, blockStart, blockStart, [])
if params.lang == langNone: if params.lang == langNone:
if len(params.langStr) > 0: if len(params.langStr) > 0:
d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr) rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage,
params.langStr)
for letter in m.text: escChar(d.target, result, letter, emText) for letter in m.text: escChar(d.target, result, letter, emText)
else: else:
renderCodeLang(result, params.lang, m.text, d.target) renderCodeLang(result, params.lang, m.text, d.target)
@ -1564,23 +1565,24 @@ proc rstToHtml*(s: string, options: RstParseOptions,
result = "" result = ""
const filen = "input" const filen = "input"
let (rst, filenames, _) = rstParse(s, filen,
line=LineRstInit, column=ColRstInit,
options, myFindFile, msgHandler)
var d: RstGenerator var d: RstGenerator
initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler) initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames)
var dummyHasToc = false
var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit,
dummyHasToc, options, myFindFile, msgHandler)
result = "" result = ""
renderRstToOut(d, rst, result) renderRstToOut(d, rst, result)
strbasics.strip(result)
proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} = proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} =
## Convenience proc for `renderRstToOut` and `initRstGenerator`. ## Convenience proc for `renderRstToOut` and `initRstGenerator`.
runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}""" runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}"""
if rstSource.len == 0: return if rstSource.len == 0: return
var option: bool let (rst, filenames, _) = rstParse(rstSource, "",
line=LineRstInit, column=ColRstInit,
options)
var rstGenera: RstGenerator var rstGenera: RstGenerator
rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", options) rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", filenames=filenames)
rstGenera.renderRstToOut( rstGenera.renderRstToOut(rst, result)
rstParse(rstSource, "", line=LineRstInit, column=ColRstInit, strbasics.strip(result)
option, options),
result)

58
tests/stdlib/trst.nim
View file

@ -5,6 +5,8 @@ discard """
[Suite] RST indentation [Suite] RST indentation
[Suite] Warnings
[Suite] RST include directive [Suite] RST include directive
[Suite] RST escaping [Suite] RST escaping
@ -51,9 +53,8 @@ proc toAst(input: string,
# we don't find any files in online mode: # we don't find any files in online mode:
result = "" result = ""
var dummyHasToc = false var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit, rstOptions, myFindFile, testMsgHandler)
dummyHasToc, rstOptions, myFindFile, testMsgHandler)
result = renderRstToStr(rst) result = renderRstToStr(rst)
except EParseError as e: except EParseError as e:
if e.msg != "": if e.msg != "":
@ -356,6 +357,53 @@ suite "RST indentation":
# "template..." should be parsed as a definition list attached to ":test:": # "template..." should be parsed as a definition list attached to ":test:":
check inputWrong.toAst != ast check inputWrong.toAst != ast
suite "Warnings":
test "warnings for broken footnotes/links/substitutions":
let input = dedent"""
firstParagraph
footnoteRef [som]_
link `a broken Link`_
substitution |undefined subst|
link short.link_
lastParagraph
"""
var warnings = new seq[string]
let output = input.toAst(warnings=warnings)
check(warnings[] == @[
"input(3, 14) Warning: broken link 'citation-som'",
"input(5, 7) Warning: broken link 'a-broken-link'",
"input(7, 15) Warning: unknown substitution 'undefined subst'",
"input(9, 6) Warning: broken link 'shortdotlink'"
])
test "With include directive and blank lines at the beginning":
"other.rst".writeFile(dedent"""
firstParagraph
here brokenLink_""")
let input = ".. include:: other.rst"
var warnings = new seq[string]
let output = input.toAst(warnings=warnings)
check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"]
check(output == dedent"""
rnInner
rnParagraph
rnLeaf 'firstParagraph'
rnParagraph
rnLeaf 'here'
rnLeaf ' '
rnRef
rnLeaf 'brokenLink'
""")
removeFile("other.rst")
suite "RST include directive": suite "RST include directive":
test "Include whole": test "Include whole":
"other.rst".writeFile("**test1**") "other.rst".writeFile("**test1**")
@ -374,7 +422,7 @@ OtherStart
.. include:: other.rst .. include:: other.rst
:start-after: OtherStart :start-after: OtherStart
""" """
doAssert "<em>Visible</em>" == rstTohtml(input, {}, defaultConfig()) check "<em>Visible</em>" == rstTohtml(input, {}, defaultConfig())
removeFile("other.rst") removeFile("other.rst")
test "Include everything before": test "Include everything before":
@ -406,7 +454,7 @@ And this should **NOT** be visible in `docs.html`
:start-after: OtherStart :start-after: OtherStart
:end-before: OtherEnd :end-before: OtherEnd
""" """
doAssert "<em>Visible</em>" == rstTohtml(input, {}, defaultConfig()) check "<em>Visible</em>" == rstTohtml(input, {}, defaultConfig())
removeFile("other.rst") removeFile("other.rst")

120
tests/stdlib/trstgen.nim
View file

@ -237,8 +237,7 @@ not in table"""
<tr><td>D1 <tt class="docutils literal"><span class="pre">""" & id"code" & " " & op"\|" & """</span></tt></td><td>D2</td></tr> <tr><td>D1 <tt class="docutils literal"><span class="pre">""" & id"code" & " " & op"\|" & """</span></tt></td><td>D2</td></tr>
<tr><td>E1 | text</td><td></td></tr> <tr><td>E1 | text</td><td></td></tr>
<tr><td></td><td>F2 without pipe</td></tr> <tr><td></td><td>F2 without pipe</td></tr>
</table><p>not in table</p> </table><p>not in table</p>""")
""")
let input2 = """ let input2 = """
| A1 header | A2 | | A1 header | A2 |
| --- | --- |""" | --- | --- |"""
@ -420,9 +419,10 @@ Some chapter
"the following intermediate section level(s) are missing on " & "the following intermediate section level(s) are missing on " &
"lines 12..15: underline -----)") "lines 12..15: underline -----)")
test "RST sections overline":
# the same as input9good but with overline headings # the same as input9good but with overline headings
# first overline heading has a special meaning: document title # first overline heading has a special meaning: document title
let input10 = dedent """ let input = dedent """
====== ======
Title0 Title0
====== ======
@ -454,22 +454,23 @@ Some chapter
~~~~~ ~~~~~
""" """
var option: bool
var rstGenera: RstGenerator var rstGenera: RstGenerator
var output10: string var output: string
rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {}) let (rst, files, _) = rstParse(input, "", 1, 1, {})
rstGenera.renderRstToOut(rstParse(input10, "", 1, 1, option, {}), output10) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files)
rstGenera.renderRstToOut(rst, output)
doAssert rstGenera.meta[metaTitle] == "Title0" doAssert rstGenera.meta[metaTitle] == "Title0"
doAssert rstGenera.meta[metaSubTitle] == "SubTitle0" doAssert rstGenera.meta[metaSubTitle] == "SubTitle0"
doAssert "<h1 id=\"level1\"><center>Level1</center></h1>" in output10 doAssert "<h1 id=\"level1\"><center>Level1</center></h1>" in output
doAssert "<h2 id=\"level2\">Level2</h2>" in output10 doAssert "<h2 id=\"level2\">Level2</h2>" in output
doAssert "<h3 id=\"level3\"><center>Level3</center></h3>" in output10 doAssert "<h3 id=\"level3\"><center>Level3</center></h3>" in output
doAssert "<h1 id=\"l1\"><center>L1</center></h1>" in output10 doAssert "<h1 id=\"l1\"><center>L1</center></h1>" in output
doAssert "<h2 id=\"another2\">Another2</h2>" in output10 doAssert "<h2 id=\"another2\">Another2</h2>" in output
doAssert "<h3 id=\"more3\"><center>More3</center></h3>" in output10 doAssert "<h3 id=\"more3\"><center>More3</center></h3>" in output
test "RST sections overline 2":
# check that a paragraph prevents interpreting overlines as document titles # check that a paragraph prevents interpreting overlines as document titles
let input11 = dedent """ let input = dedent """
Paragraph Paragraph
====== ======
@ -480,18 +481,19 @@ Some chapter
SubTitle0 SubTitle0
+++++++++ +++++++++
""" """
var option11: bool var rstGenera: RstGenerator
var rstGenera11: RstGenerator var output: string
var output11: string let (rst, files, _) = rstParse(input, "", 1, 1, {})
rstGenera11.initRstGenerator(outHtml, defaultConfig(), "input", {}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
rstGenera11.renderRstToOut(rstParse(input11, "", 1, 1, option11, {}), output11) rstGenera.renderRstToOut(rst, output)
doAssert rstGenera11.meta[metaTitle] == "" doAssert rstGenera.meta[metaTitle] == ""
doAssert rstGenera11.meta[metaSubTitle] == "" doAssert rstGenera.meta[metaSubTitle] == ""
doAssert "<h1 id=\"title0\"><center>Title0</center></h1>" in output11 doAssert "<h1 id=\"title0\"><center>Title0</center></h1>" in output
doAssert "<h2 id=\"subtitle0\"><center>SubTitle0</center></h2>" in output11 doAssert "<h2 id=\"subtitle0\"><center>SubTitle0</center></h2>" in output
test "RST+Markdown sections":
# check that RST and Markdown headings don't interfere # check that RST and Markdown headings don't interfere
let input12 = dedent """ let input = dedent """
====== ======
Title0 Title0
====== ======
@ -509,14 +511,14 @@ Some chapter
MySection2a MySection2a
----------- -----------
""" """
var option12: bool var rstGenera: RstGenerator
var rstGenera12: RstGenerator var output: string
var output12: string let (rst, files, _) = rstParse(input, "", 1, 1, {roSupportMarkdown})
rstGenera12.initRstGenerator(outHtml, defaultConfig(), "input", {}) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
rstGenera12.renderRstToOut(rstParse(input12, "", 1, 1, option12, {roSupportMarkdown}), output12) rstGenera.renderRstToOut(rst, output)
doAssert rstGenera12.meta[metaTitle] == "Title0" doAssert rstGenera.meta[metaTitle] == "Title0"
doAssert rstGenera12.meta[metaSubTitle] == "" doAssert rstGenera.meta[metaSubTitle] == ""
doAssert output12 == doAssert output ==
"\n<h1 id=\"mysection1a\">MySection1a</h1>" & # RST "\n<h1 id=\"mysection1a\">MySection1a</h1>" & # RST
"\n<h1 id=\"mysection1b\">MySection1b</h1>" & # Markdown "\n<h1 id=\"mysection1b\">MySection1b</h1>" & # Markdown
"\n<h1 id=\"mysection1c\">MySection1c</h1>" & # RST "\n<h1 id=\"mysection1c\">MySection1c</h1>" & # RST
@ -628,7 +630,7 @@ let x = 1
let p2 = """<p>Par2 <tt class="docutils literal"><span class="pre">value2</span></tt>.</p>""" let p2 = """<p>Par2 <tt class="docutils literal"><span class="pre">value2</span></tt>.</p>"""
let p3 = """<p>Par3 <tt class="docutils literal"><span class="pre">""" & id"value3" & "</span></tt>.</p>" let p3 = """<p>Par3 <tt class="docutils literal"><span class="pre">""" & id"value3" & "</span></tt>.</p>"
let p4 = """<p>Par4 <tt class="docutils literal"><span class="pre">value4</span></tt>.</p>""" let p4 = """<p>Par4 <tt class="docutils literal"><span class="pre">value4</span></tt>.</p>"""
let expected = p1 & p2 & "\n" & p3 & "\n" & p4 & "\n" let expected = p1 & p2 & "\n" & p3 & "\n" & p4
check(input.toHtml == expected) check(input.toHtml == expected)
test "role directive": test "role directive":
@ -653,8 +655,8 @@ Check that comment disappears:
let output1 = input1.toHtml let output1 = input1.toHtml
doAssert output1 == "Check that comment disappears:" doAssert output1 == "Check that comment disappears:"
test "RST line blocks": test "RST line blocks + headings":
let input1 = """ let input = """
===== =====
Test1 Test1
===== =====
@ -665,19 +667,20 @@ Test1
| other line | other line
""" """
var option: bool
var rstGenera: RstGenerator var rstGenera: RstGenerator
var output1: string var output: string
rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {}) let (rst, files, _) = rstParse(input, "", 1, 1, {})
rstGenera.renderRstToOut(rstParse(input1, "", 1, 1, option, {}), output1) rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
rstGenera.renderRstToOut(rst, output)
doAssert rstGenera.meta[metaTitle] == "Test1" doAssert rstGenera.meta[metaTitle] == "Test1"
# check that title was not overwritten to '|' # check that title was not overwritten to '|'
doAssert output1 == "<p><br/><br/>line block<br/>other line<br/></p>" doAssert output == "<p><br/><br/>line block<br/>other line<br/></p>"
let output1l = rstToLatex(input1, {}) let output1l = rstToLatex(input, {})
doAssert "line block\n\n" in output1l doAssert "line block\n\n" in output1l
doAssert "other line\n\n" in output1l doAssert "other line\n\n" in output1l
doAssert output1l.count("\\vspace") == 2 + 2 # +2 surrounding paddings doAssert output1l.count("\\vspace") == 2 + 2 # +2 surrounding paddings
test "RST line blocks":
let input2 = dedent""" let input2 = dedent"""
Paragraph1 Paragraph1
@ -686,7 +689,7 @@ Test1
Paragraph2""" Paragraph2"""
let output2 = input2.toHtml let output2 = input2.toHtml
doAssert "Paragraph1<p><br/></p> <p>Paragraph2</p>\n" == output2 doAssert "Paragraph1<p><br/></p> <p>Paragraph2</p>" == output2
let input3 = dedent""" let input3 = dedent"""
| xxx | xxx
@ -853,7 +856,7 @@ Test1
check("input(6, 1) Warning: RST style: \n" & check("input(6, 1) Warning: RST style: \n" &
"not enough indentation on line 6" in warnings8[0]) "not enough indentation on line 6" in warnings8[0])
doAssert output8 == "Paragraph.<ol class=\"upperalpha simple\">" & doAssert output8 == "Paragraph.<ol class=\"upperalpha simple\">" &
"<li>stringA</li>\n<li>stringB</li>\n</ol>\n<p>C. string1 string2 </p>\n" "<li>stringA</li>\n<li>stringB</li>\n</ol>\n<p>C. string1 string2 </p>"
test "Markdown enumerated lists": test "Markdown enumerated lists":
let input1 = dedent """ let input1 = dedent """
@ -1034,9 +1037,7 @@ Test1
""" """
var warnings8 = new seq[string] var warnings8 = new seq[string]
let output8 = input8.toHtml(warnings=warnings8) let output8 = input8.toHtml(warnings=warnings8)
# TODO: the line 1 is arbitrary because reference lines are not preserved check(warnings8[] == @["input(3, 7) Warning: broken link 'citation-som'"])
check(warnings8[] == @["input(1, 1) Warning: unknown substitution " &
"\'citation-som\'"])
# check that footnote group does not break parsing of other directives: # check that footnote group does not break parsing of other directives:
let input9 = dedent """ let input9 = dedent """
@ -1144,10 +1145,33 @@ Test1
""" """
var error = new string var error = new string
let output = input.toHtml(error=error) let output = input.toHtml(error=error)
check(error[] == "input(1, 1) Error: invalid field: " & check(error[] == "input(2, 3) Error: invalid field: " &
"extra arguments were given to number-lines: ' let a = 1'") "extra arguments were given to number-lines: ' let a = 1'")
check "" == output check "" == output
test "code-block warning":
let input = dedent """
.. code:: Nim
:unsupportedField: anything
.. code:: unsupportedLang
anything
```anotherLang
someCode
```
"""
let warnings = new seq[string]
let output = input.toHtml(warnings=warnings)
check(warnings[] == @[
"input(2, 4) Warning: field 'unsupportedField' not supported",
"input(4, 11) Warning: language 'unsupportedLang' not supported",
"input(8, 4) Warning: language 'anotherLang' not supported"
])
check(output == "<pre class = \"listing\">anything</pre>" &
"<p><pre class = \"listing\">\nsomeCode\n</pre> </p>")
test "RST admonitions": test "RST admonitions":
# check that all admonitions are implemented # check that all admonitions are implemented
let input0 = dedent """ let input0 = dedent """
@ -1477,7 +1501,7 @@ Test1
check "(3, 15) Warning: " in warnings[1] check "(3, 15) Warning: " in warnings[1]
check "language 'py:class' not supported" in warnings[1] check "language 'py:class' not supported" in warnings[1]
check("""<p>See function <span class="py:func">spam</span>.</p>""" & "\n" & check("""<p>See function <span class="py:func">spam</span>.</p>""" & "\n" &
"""<p>See also <span class="py:class">egg</span>. </p>""" & "\n" == """<p>See also <span class="py:class">egg</span>. </p>""" ==
output) output)
test "(not) Roles: check escaping 1": test "(not) Roles: check escaping 1":