RST heading improvements (fix #17091) (#17195)

compiler/docgen.nim

@@ -1237,6 +1237,10 @@ proc genOutFile(d: PDoc, groupedToc = false): Rope =
     # Modules get an automatic title for the HTML, but no entry in the index.
     # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups
     title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("")
+  var subtitle = "".rope
+  if d.meta[metaSubtitle] != "":
+    dispA(d.conf, subtitle, "<h2 class=\"subtitle\">$1</h2>",
+        "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle].rope])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
   let bodyname = if d.hasToc and not d.isPureRst:
@@ -1245,18 +1249,18 @@ proc genOutFile(d: PDoc, groupedToc = false): Rope =
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
   let seeSrcRope = genSeeSrcRope(d, d.filename, 1)
-  content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
+  content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title", "subtitle",
       "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref", "body_toc_groupsection", "seeSrc"],
-      [title.rope, toc, d.modDesc, rope(getDateStr()),
+      [title.rope, subtitle, toc, d.modDesc, rope(getDateStr()),
       rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile.AbsoluteFile, theindexFname.RelativeFile), groupsection.rope, seeSrcRope])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "subtitle", "tableofcontents", "moduledesc", "date", "time",
         "content", "author", "version", "analytics", "deprecationMsg"],
         [relLink(d.conf.outDir, d.destFile.AbsoluteFile, nimdocOutCss.RelativeFile),
         relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile),
-        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        title.rope, subtitle, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
         content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
   else:
     code = content
@@ -1408,11 +1412,11 @@ proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"")
 
   let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
       "nimdoccss", "dochackjs",
-      "title", "tableofcontents", "moduledesc", "date", "time",
+      "title", "subtitle", "tableofcontents", "moduledesc", "date", "time",
       "content", "author", "version", "analytics"],
       [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
       relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
-      rope"Index", nil, nil, rope(getDateStr()),
+      rope"Index", rope"", nil, nil, rope(getDateStr()),
       rope(getClockStr()), content, nil, nil, nil])
   # no analytics because context is not available
 

config/nimdoc.cfg

@@ -281,7 +281,7 @@ window.addEventListener('DOMContentLoaded', main);
 <body>
 <div class="document" id="documentId">
   <div class="container">
-    <h1 class="title">$title</h1>
+    <h1 class="title">$title</h1>$subtitle
     $content
     <div class="row">
       <div class="twelve-columns footer">

config/nimdoc.tex.cfg

@@ -62,7 +62,7 @@ rightline=false,
 bottomline=false}
 
 \begin{document}
-\title{$title $version}
+\title{$title $version $subtitle}
 \author{$author}
 
 \tolerance 1414 

doc/nimdoc.css

@@ -384,6 +384,7 @@ h2 {
   margin-top: 2em; }
 
 h2.subtitle {
+  margin-top: 0em;
   text-align: center; }
 
 h3 {

lib/packages/docutils/rst.nim

@@ -8,9 +8,13 @@
 #
 
 ## ==================================
-## rst: Nim-flavored reStructuredText
+##                rst
 ## ==================================
 ##
+## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## Nim-flavored reStructuredText
+## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+##
 ## This module implements a `reStructuredText`:idx: (RST) parser.
 ## A large subset is implemented with some limitations_ and
 ## `Nim-specific features`_.
@@ -410,7 +414,14 @@ proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int =
     tokens[0].kind = tkIndent
 
 type
-  LevelMap = array[char, int]
+  LevelInfo = object
+    symbol: char         # adornment character
+    hasOverline: bool    # has also overline (besides underline)?
+    line: int            # the last line of this style occurrence
+                         # (for error message)
+    hasPeers: bool       # has headings on the same level of hierarchy?
+  LevelMap = seq[LevelInfo]   # Saves for each possible title adornment
+                              # style its level in the current document.
   Substitution = object
     key*: string
     value*: PRstNode
@@ -433,7 +444,10 @@ type
 
   SharedState = object
     options: RstParseOptions    # parsing options
-    uLevel, oLevel: int         # counters for the section levels
+    hLevels: LevelMap           # hierarchy of heading styles
+    hTitleCnt: int              # =0 if no title, =1 if only main title,
+                                # =2 if both title and subtitle are present
+    hCurLevel: int              # current section level
     subs: seq[Substitution]     # substitutions
     refs: seq[Substitution]     # references
     anchors: seq[AnchorSubst]   # internal target substitutions
@@ -443,14 +457,6 @@ type
     lineFootnoteSymRef: seq[int]  # footnote line, their reference [*]_
     footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
                                   # number, order of occurrence
-    underlineToLevel: LevelMap  # Saves for each possible title adornment
-                                # character its level in the
-                                # current document.
-                                # This is for single underline adornments.
-    overlineToLevel: LevelMap   # Saves for each possible title adornment
-                                # character its level in the current
-                                # document.
-                                # This is for over-underline adornments.
     msgHandler: MsgHandler      # How to handle errors.
     findFile: FindFileHandler   # How to find files.
 
@@ -1408,11 +1414,30 @@ proc parseLiteralBlock(p: var RstParser): PRstNode =
       inc p.idx
   result.add(n)
 
-proc getLevel(map: var LevelMap, lvl: var int, c: char): int =
+proc getLevel(p: var RstParser, c: char, hasOverline: bool): int =
-  if map[c] == 0:
+  ## Returns (preliminary) heading level corresponding to `c` and
-    inc lvl
+  ## `hasOverline`. If level does not exist, add it first.
-    map[c] = lvl
+  for i, hType in p.s.hLevels:
-  result = map[c]
+    if hType.symbol == c and hType.hasOverline == hasOverline:
+      p.s.hLevels[i].line = curLine(p)
+      p.s.hLevels[i].hasPeers = true
+      return i
+  p.s.hLevels.add LevelInfo(symbol: c, hasOverline: hasOverline,
+                            line: curLine(p), hasPeers: false)
+  result = p.s.hLevels.len - 1
+
+proc countTitles(p: var RstParser, n: PRstNode) =
+  ## Fill `p.s.hTitleCnt`
+  for node in n.sons:
+    if node != nil:
+      if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}:
+        break
+      if node.kind == rnOverline:
+        if p.s.hLevels[p.s.hTitleCnt].hasPeers:
+          break
+        inc p.s.hTitleCnt
+        if p.s.hTitleCnt >= 2:
+          break
 
 proc tokenAfterNewline(p: RstParser): int =
   result = p.idx
@@ -1529,7 +1554,7 @@ proc whichSection(p: RstParser): RstNodeKind =
       result = rnLeaf
   of tkPunct:
     if isMarkdownHeadline(p):
-      result = rnHeadline
+      result = rnMarkdownHeadline
     elif roSupportMarkdown in p.s.options and predNL(p) and
         match(p, p.idx, "| w") and findPipe(p, p.idx+3):
       result = rnMarkdownTable
@@ -1599,7 +1624,8 @@ proc parseParagraph(p: var RstParser, result: PRstNode) =
       elif currentTok(p).ival == currInd(p):
         inc p.idx
         case whichSection(p)
-        of rnParagraph, rnLeaf, rnHeadline, rnOverline, rnDirective:
+        of rnParagraph, rnLeaf, rnHeadline, rnMarkdownHeadline,
+            rnOverline, rnDirective:
           result.add newLeaf(" ")
         of rnLineBlock:
           result.addIfNotNil(parseLineBlock(p))
@@ -1620,20 +1646,60 @@ proc parseParagraph(p: var RstParser, result: PRstNode) =
       parseInline(p, result)
     else: break
 
+proc checkHeadingHierarchy(p: RstParser, lvl: int) =
+  if lvl - p.s.hCurLevel > 1:  # broken hierarchy!
+    proc descr(l: int): string =
+      (if p.s.hLevels[l].hasOverline: "overline " else: "underline ") &
+      repeat(p.s.hLevels[l].symbol, 5)
+    var msg = "(section level inconsistent: "
+    msg.add descr(lvl) & " unexpectedly found, " &
+      "while the following intermediate section level(s) are missing on lines "
+    msg.add $p.s.hLevels[p.s.hCurLevel].line & ".." & $curLine(p) & ":"
+    for l in p.s.hCurLevel+1 .. lvl-1:
+      msg.add " " & descr(l)
+      if l != lvl-1: msg.add ","
+    rstMessage(p, meNewSectionExpected, msg & ")")
+
 proc parseHeadline(p: var RstParser): PRstNode =
-  result = newRstNode(rnHeadline)
   if isMarkdownHeadline(p):
+    result = newRstNode(rnMarkdownHeadline)
+    # Note that level hierarchy is not checked for markdown headings
     result.level = currentTok(p).symbol.len
     assert(nextTok(p).kind == tkWhite)
     inc p.idx, 2
     parseUntilNewline(p, result)
   else:
+    result = newRstNode(rnHeadline)
     parseUntilNewline(p, result)
     assert(currentTok(p).kind == tkIndent)
     assert(nextTok(p).kind == tkAdornment)
     var c = nextTok(p).symbol[0]
     inc p.idx, 2
-    result.level = getLevel(p.s.underlineToLevel, p.s.uLevel, c)
+    result.level = getLevel(p, c, hasOverline=false)
+    checkHeadingHierarchy(p, result.level)
+    p.s.hCurLevel = result.level
+  addAnchor(p, rstnodeToRefname(result), reset=true)
+
+proc parseOverline(p: var RstParser): PRstNode =
+  var c = currentTok(p).symbol[0]
+  inc p.idx, 2
+  result = newRstNode(rnOverline)
+  while true:
+    parseUntilNewline(p, result)
+    if currentTok(p).kind == tkIndent:
+      inc p.idx
+      if prevTok(p).ival > currInd(p):
+        result.add newLeaf(" ")
+      else:
+        break
+    else:
+      break
+  result.level = getLevel(p, c, hasOverline=true)
+  checkHeadingHierarchy(p, result.level)
+  p.s.hCurLevel = result.level
+  if currentTok(p).kind == tkAdornment:
+    inc p.idx
+    if currentTok(p).kind == tkIndent: inc p.idx
   addAnchor(p, rstnodeToRefname(result), reset=true)
 
 type
@@ -1779,26 +1845,6 @@ proc parseTransition(p: var RstParser): PRstNode =
   if currentTok(p).kind == tkIndent: inc p.idx
   if currentTok(p).kind == tkIndent: inc p.idx
 
-proc parseOverline(p: var RstParser): PRstNode =
-  var c = currentTok(p).symbol[0]
-  inc p.idx, 2
-  result = newRstNode(rnOverline)
-  while true:
-    parseUntilNewline(p, result)
-    if currentTok(p).kind == tkIndent:
-      inc p.idx
-      if prevTok(p).ival > currInd(p):
-        result.add newLeaf(" ")
-      else:
-        break
-    else:
-      break
-  result.level = getLevel(p.s.overlineToLevel, p.s.oLevel, c)
-  if currentTok(p).kind == tkAdornment:
-    inc p.idx                # XXX: check?
-    if currentTok(p).kind == tkIndent: inc p.idx
-  addAnchor(p, rstnodeToRefname(result), reset=true)
-
 proc parseBulletList(p: var RstParser): PRstNode =
   result = nil
   if nextTok(p).kind == tkWhite:
@@ -1982,7 +2028,7 @@ proc parseSection(p: var RstParser, result: PRstNode) =
       if p.idx > 0: dec p.idx
       a = parseFields(p)
     of rnTransition: a = parseTransition(p)
-    of rnHeadline: a = parseHeadline(p)
+    of rnHeadline, rnMarkdownHeadline: a = parseHeadline(p)
     of rnOverline: a = parseOverline(p)
     of rnTable: a = parseSimpleTable(p)
     of rnMarkdownTable: a = parseMarkdownTable(p)
@@ -2399,6 +2445,15 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode =
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
       else: rstMessage(p, mwUnknownSubstitution, key)
+  of rnHeadline, rnOverline:
+    # fix up section levels depending on presence of a title and subtitle
+    if p.s.hTitleCnt == 2:
+      if n.level == 1:    # it's the subtitle
+        n.level = 0
+      elif n.level >= 2:  # normal sections
+        n.level -= 1
+    elif p.s.hTitleCnt == 0:
+      n.level += 1
   of rnRef:
     let refn = rstnodeToRefname(n)
     var y = findRef(p, refn)
@@ -2498,6 +2553,7 @@ proc rstParse*(text, filename: string,
   p.line = line
   p.col = column + getTokens(text, roSkipPounds in options, p.tok)
   let unresolved = parseDoc(p)
+  countTitles(p, unresolved)
   orderFootnotes(p)
   result = resolveSubs(p, unresolved)
   hasToc = p.hasToc

lib/packages/docutils/rstast.nim

@@ -18,6 +18,7 @@ type
     rnInner,                  # an inner node or a root
     rnHeadline,               # a headline
     rnOverline,               # an over- and underlined headline
+    rnMarkdownHeadline,       # a Markdown headline
     rnTransition,             # a transition (the ------------- <hr> thingie)
     rnParagraph,              # a paragraph
     rnBulletList,             # a bullet list
@@ -84,9 +85,10 @@ type
     of rnAdmonition:
       adType*: string         ## admonition type: "note", "caution", etc. This
                               ## text will set the style and also be displayed
-    of rnOverline, rnHeadline:
+    of rnOverline, rnHeadline, rnMarkdownHeadline:
-      level*: int             ## level of headings starting from 1 (document
+      level*: int             ## level of headings starting from 1 (main
-                              ## title) to larger ones (minor sub-sections)
+                              ## chapter) to larger ones (minor sub-sections)
+                              ## level=0 means it's document title or subtitle
     of rnFootnote, rnCitation, rnFootnoteRef:
       order*: int             ## footnote order (for auto-symbol footnotes and
                               ## auto-numbered ones without a label)
@@ -363,8 +365,8 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
     if node.lineIndent == "\n": txt = "\t(blank line)"
     else: txt = "\tlineIndent=" & $node.lineIndent.len
     result.add txt
-  of rnHeadline, rnOverline:
+  of rnHeadline, rnOverline, rnMarkdownHeadline:
-    result.add (if node.level == 0: "" else: "\tlevel=" & $node.level)
+    result.add "\tlevel=" & $node.level
   of rnFootnote, rnCitation, rnFootnoteRef:
     result.add (if node.order == 0:   "" else: "\torder=" & $node.order)
   else:

lib/packages/docutils/rstgen.nim

@@ -797,11 +797,11 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
     spaces(max(0, n.level)) & tmp)
 
 proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
-  if d.meta[metaTitle].len == 0:
+  if n.level == 0 and d.meta[metaTitle].len == 0:
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaTitle])
     d.currentSection = d.meta[metaTitle]
-  elif d.meta[metaSubtitle].len == 0:
+  elif n.level == 0 and d.meta[metaSubtitle].len == 0:
     for i in countup(0, len(n)-1):
       renderRstToOut(d, n.sons[i], d.meta[metaSubtitle])
     d.currentSection = d.meta[metaSubtitle]
@@ -1140,7 +1140,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   if n == nil: return
   case n.kind
   of rnInner: renderAux(d, n, result)
-  of rnHeadline: renderHeadline(d, n, result)
+  of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result)
   of rnOverline: renderOverline(d, n, result)
   of rnTransition: renderAux(d, n, "<hr$2 />\n", "\\hrule$2\n", result)
   of rnParagraph: renderAux(d, n, "<p$2>$1</p>\n", "$2\n$1\n\n", result)

nimdoc/testproject/expected/nimdoc.out.css

@@ -384,6 +384,7 @@ h2 {
   margin-top: 2em; }
 
 h2.subtitle {
+  margin-top: 0em;
   text-align: center; }
 
 h3 {

tests/stdlib/trstgen.nim

@@ -298,6 +298,162 @@ Some chapter
     expect(EParseError):
       let output8 = rstToHtml(input8, {roSupportMarkdown}, defaultConfig())
 
+    # check that hierarchy of title styles works
+    let input9good = dedent """
+      Level1
+      ======
+
+      Level2
+      ------
+
+      Level3
+      ~~~~~~
+
+      L1
+      ==
+
+      Another2
+      --------
+
+      More3
+      ~~~~~
+
+      """
+    let output9good = rstToHtml(input9good, {roSupportMarkdown}, defaultConfig())
+    doAssert "<h1 id=\"level1\">Level1</h1>" in output9good
+    doAssert "<h2 id=\"level2\">Level2</h2>" in output9good
+    doAssert "<h3 id=\"level3\">Level3</h3>" in output9good
+    doAssert "<h1 id=\"l1\">L1</h1>" in output9good
+    doAssert "<h2 id=\"another2\">Another2</h2>" in output9good
+    doAssert "<h3 id=\"more3\">More3</h3>" in output9good
+
+    # check that swap causes an exception
+    let input9Bad = dedent """
+      Level1
+      ======
+
+      Level2
+      ------
+      
+      Level3
+      ~~~~~~
+
+      L1
+      ==
+
+      More
+      ~~~~
+      
+      Another
+      -------
+
+      """
+    expect(EParseError):
+      let output9Bad = rstToHtml(input9Bad, {roSupportMarkdown}, defaultConfig())
+
+    # the same as input9good but with overline headings
+    # first overline heading has a special meaning: document title
+    let input10 = dedent """
+      ======
+      Title0
+      ======
+
+      +++++++++
+      SubTitle0
+      +++++++++
+
+      ------
+      Level1
+      ------
+
+      Level2
+      ------
+
+      ~~~~~~
+      Level3
+      ~~~~~~
+
+      --
+      L1
+      --
+
+      Another2
+      --------
+
+      ~~~~~
+      More3
+      ~~~~~
+
+      """
+    var option: bool
+    var rstGenera: RstGenerator
+    var output10: string
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {})
+    rstGenera.renderRstToOut(rstParse(input10, "", 1, 1, option, {}), output10)
+    doAssert rstGenera.meta[metaTitle] == "Title0"
+    doAssert rstGenera.meta[metaSubTitle] == "SubTitle0"
+    doAssert "<h1 id=\"level1\"><center>Level1</center></h1>" in output10
+    doAssert "<h2 id=\"level2\">Level2</h2>" in output10
+    doAssert "<h3 id=\"level3\"><center>Level3</center></h3>" in output10
+    doAssert "<h1 id=\"l1\"><center>L1</center></h1>" in output10
+    doAssert "<h2 id=\"another2\">Another2</h2>" in output10
+    doAssert "<h3 id=\"more3\"><center>More3</center></h3>" in output10
+
+    # check that a paragraph prevents interpreting overlines as document titles
+    let input11 = dedent """
+      Paragraph
+
+      ======
+      Title0
+      ======
+
+      +++++++++
+      SubTitle0
+      +++++++++
+      """
+    var option11: bool
+    var rstGenera11: RstGenerator
+    var output11: string
+    rstGenera11.initRstGenerator(outHtml, defaultConfig(), "input", {})
+    rstGenera11.renderRstToOut(rstParse(input11, "", 1, 1, option11, {}), output11)
+    doAssert rstGenera11.meta[metaTitle] == ""
+    doAssert rstGenera11.meta[metaSubTitle] == ""
+    doAssert "<h1 id=\"title0\"><center>Title0</center></h1>" in output11
+    doAssert "<h2 id=\"subtitle0\"><center>SubTitle0</center></h2>" in output11
+
+    # check that RST and Markdown headings don't interfere
+    let input12 = dedent """
+      ======
+      Title0
+      ======
+
+      MySection1a
+      +++++++++++
+
+      # MySection1b
+
+      MySection1c
+      +++++++++++
+
+      ##### MySection5a
+
+      MySection2a
+      -----------
+      """
+    var option12: bool
+    var rstGenera12: RstGenerator
+    var output12: string
+    rstGenera12.initRstGenerator(outHtml, defaultConfig(), "input", {})
+    rstGenera12.renderRstToOut(rstParse(input12, "", 1, 1, option12, {roSupportMarkdown}), output12)
+    doAssert rstGenera12.meta[metaTitle] == "Title0"
+    doAssert rstGenera12.meta[metaSubTitle] == ""
+    doAssert output12 ==
+             "\n<h1 id=\"mysection1a\">MySection1a</h1>" & # RST
+             "\n<h1 id=\"mysection1b\">MySection1b</h1>" & # Markdown
+             "\n<h1 id=\"mysection1c\">MySection1c</h1>" & # RST
+             "\n<h5 id=\"mysection5a\">MySection5a</h5>" & # Markdown
+             "\n<h2 id=\"mysection2a\">MySection2a</h2>"   # RST
+
   test "RST inline text":
     let input1 = "GC_step"
     let output1 = input1.toHtml