Rename invalidrst -> minvalidrst

changelog.md

@@ -138,3 +138,5 @@
 
 ## Tool changes
 
+- The `nim doc` command now supports ignoring RST errors when parsing
+  by using the `--docIgnoreParseErrors` flag

compiler/commands.nim

@@ -437,6 +437,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.docSeeSrcUrl = arg
   of "docroot":
     conf.docRoot = if arg.len == 0: "@default" else: arg
+  of "docignoreparseerrors":
+    conf.docIgnoreParseErrors = true
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":

compiler/docgen.nim

@@ -146,8 +146,13 @@ proc parseRst(text, filename: string,
               rstOptions: RstParseOptions;
               conf: ConfigRef): PRstNode =
   declareClosures()
+  var msgHandler: MsgHandler
+  if conf.docIgnoreParseErrors:
+    msgHandler = rst.noExceptionMsgHandler
+  else:
+    msgHandler = compilerMsgHandler
   result = rstParse(text, filename, line, column, hasToc, rstOptions,
-                    docgenFindFile, compilerMsgHandler)
+                    docgenFindFile, msgHandler)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile =
@@ -170,9 +175,14 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.module = module
   result.conf = conf
   result.cache = cache
+  var msgHandler: MsgHandler
+  if conf.docIgnoreParseErrors:
+    msgHandler = rst.noExceptionMsgHandler
+  else:
+    msgHandler = defaultMsgHandler
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
                    conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown},
-                   docgenFindFile, compilerMsgHandler)
+                   docgenFindFile, msgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
     result.analytics = """

compiler/options.nim

@@ -274,6 +274,7 @@ type
     docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \
     # The string uses the formatting variables `path` and `line`.
     docRoot*: string ## see nim --fullhelp for --docRoot
+    docIgnoreParseErrors*: bool ## continue parsing docs if errors occur
 
      # the used compiler
     cIncludes*: seq[AbsoluteDir]  # directories to search for included files

doc/advopt.txt

@@ -79,6 +79,7 @@ Advanced options:
   --docSeeSrcUrl:url        activate 'see source' for doc and doc2 commands
                             (see doc.item.seesrc in config/nimdoc.cfg)
   --docInternal             also generate documentation for non-exported symbols
+  --docIgnoreParseErrors    continue parsing docs if rst errors occur
   --lineDir:on|off          generation of #line directive on|off
   --embedsrc:on|off         embeds the original source code as comments
                             in the generated output

lib/packages/docutils/rst.nim

@@ -312,6 +312,14 @@ proc defaultMsgHandler*(filename: string, line, col: int, msgkind: MsgKind,
   if mc == mcError: raise newException(EParseError, message)
   else: writeLine(stdout, message)
 
+proc noExceptionMsgHandler*(filename: string, line, col: int, msgkind: MsgKind,
+                      arg: string) {.procvar.} =
+  let mc = msgkind.whichMsgClass
+  let a = messages[msgkind] % arg
+  let message = "$1($2, $3) $4: $5" % [filename, $line, $col, $mc, a]
+  if mc == mcError: writeLine(stderr, message)
+  else: writeLine(stdout, message)
+
 proc defaultFindFile*(filename: string): string {.procvar.} =
   if existsFile(filename): result = filename
   else: result = ""
@@ -755,13 +763,23 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
       inc(p.idx)
       if p.tok[p.idx].kind == tkIndent:
         rstMessage(p, meExpected, postfix, line, col)
+        add(father, newLeaf(p))
+        inc(p.idx)
         break
     of tkWhite:
       add(father, newRstNode(rnLeaf, " "))
       inc(p.idx)
-    else: rstMessage(p, meExpected, postfix, line, col)
+    else:
+      rstMessage(p, meExpected, postfix, line, col)
+      add(father, newLeaf(p))
+      inc(p.idx)
+      break
 
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
+  let backupIndex = p.idx
+
+  # skip ```
+  inc(p.idx)
   var args = newRstNode(rnDirArg)
   if p.tok[p.idx].kind == tkWord:
     add(args, newLeaf(p))
@@ -773,7 +791,10 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
     case p.tok[p.idx].kind
     of tkEof:
       rstMessage(p, meExpected, "```")
-      break
+      p.idx = backupIndex
+      result = newLeaf(p)
+      inc(p.idx)
+      return
     of tkPunct:
       if p.tok[p.idx].symbol == "```":
         inc(p.idx)
@@ -817,36 +838,47 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
   result = true
 
 proc parseInline(p: var RstParser, father: PRstNode) =
+  template addOrBackup(endTok) =
+    let prevTok = if (p.idx-1) < len(p.tok): p.tok[p.idx-1].symbol else: ""
+
+    if p.idx >= len(p.tok) - 1 and prevTok != endTok:
+      # we failed, so
+      p.idx = backupIndex
+      add(father, newLeaf(p))
+      inc(p.idx)
+    else:
+      add(father, n)
+
+  let backupIndex = p.idx
   case p.tok[p.idx].kind
   of tkPunct:
     if isInlineMarkupStart(p, "***"):
       var n = newRstNode(rnTripleEmphasis)
       parseUntil(p, n, "***", true)
-      add(father, n)
+      addOrBackup("***")
     elif isInlineMarkupStart(p, "**"):
       var n = newRstNode(rnStrongEmphasis)
       parseUntil(p, n, "**", true)
-      add(father, n)
+      addOrBackup("**")
     elif isInlineMarkupStart(p, "*"):
       var n = newRstNode(rnEmphasis)
       parseUntil(p, n, "*", true)
-      add(father, n)
+      addOrBackup("*")
     elif roSupportMarkdown in p.s.options and p.tok[p.idx].symbol == "```":
-      inc(p.idx)
       add(father, parseMarkdownCodeblock(p))
     elif isInlineMarkupStart(p, "``"):
       var n = newRstNode(rnInlineLiteral)
       parseUntil(p, n, "``", false)
-      add(father, n)
+      addOrBackup("``")
     elif isInlineMarkupStart(p, "`"):
       var n = newRstNode(rnInterpretedText)
       parseUntil(p, n, "`", true)
       n = parsePostfix(p, n)
-      add(father, n)
+      addOrBackup("`")
     elif isInlineMarkupStart(p, "|"):
       var n = newRstNode(rnSubstitutionReferences)
       parseUntil(p, n, "|", false)
-      add(father, n)
+      addOrBackup("|")
     elif roSupportMarkdown in p.s.options and
         p.tok[p.idx].symbol == "[" and p.tok[p.idx+1].symbol != "[" and
         parseMarkdownLink(p, father):
@@ -1773,6 +1805,7 @@ proc parseDotDot(p: var RstParser): PRstNode =
       b = dirImage(p)
     else:
       rstMessage(p, meInvalidDirective, p.tok[p.idx].symbol)
+      inc(p.idx)
     setSub(p, addNodes(a), b)
   elif match(p, p.idx, " ["):
     # footnotes, citations

tests/nimdoc/minvalidrst.nim

@@ -0,0 +1,4 @@
+proc foo*() =
+  ## a ***bunch of foo ``invalid'' breaking symbols ***
+  ## ```more symbols ** *** * | ` [
+  discard

tests/trunner.nim

@@ -6,7 +6,7 @@ discard """
 ## tests that don't quite fit the mold and are easier to handle via `execCmdEx`
 ## A few others could be added to here to simplify code.
 
-import std/[strformat,os,osproc]
+import std/[strformat,os,osproc,unittest]
 
 const nim = getCurrentCompilerExe()
 
@@ -15,8 +15,9 @@ const mode =
   elif defined(cpp): "cpp"
   else: static: doAssert false
 
-proc runCmd(file, options = ""): auto =
 const testsDir = currentSourcePath().parentDir
+
+proc runCmd(file, options = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
   doAssert fileabs.existsFile, fileabs
   let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs}"
@@ -55,11 +56,11 @@ ret=[s1:foobar s2:foobar age:25 pi:3.14]
 
 else: # don't run twice the same test
   import std/[strutils]
-  template check(msg) = doAssert msg in output, output
+  template checkrunner(msg) = doAssert msg in output, output
 
   block: # mstatic_assert
     let (output, exitCode) = runCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad")
-    check "sizeof(bool) == 2"
+    checkrunner "sizeof(bool) == 2"
     doAssert exitCode != 0
 
   block: # ABI checks
@@ -72,11 +73,11 @@ else: # don't run twice the same test
       # on platforms that support _StaticAssert natively, errors will show full context, eg:
       # error: static_assert failed due to requirement 'sizeof(unsigned char) == 8'
       # "backend & Nim disagree on size for: BadImportcType{int64} [declared in mabi_check.nim(1, 6)]"
-      check "sizeof(unsigned char) == 8"
-      check "sizeof(struct Foo2) == 1"
-      check "sizeof(Foo5) == 16"
-      check "sizeof(Foo5) == 3"
-      check "sizeof(struct Foo6) == "
+      checkrunner "sizeof(unsigned char) == 8"
+      checkrunner "sizeof(struct Foo2) == 1"
+      checkrunner "sizeof(Foo5) == 16"
+      checkrunner "sizeof(Foo5) == 3"
+      checkrunner "sizeof(struct Foo6) == "
       doAssert exitCode != 0
 
   import streams
@@ -103,3 +104,11 @@ else: # don't run twice the same test
         var (output, exitCode) = execCmdEx(cmd)
         output.stripLineEnd
         doAssert output == expected
+
+  block: # nim doc --docIgnoreParseErrors
+    let file = testsDir / "nimdoc" / "minvalidrst.nim"
+    let cmd = fmt"{nim} doc --docIgnoreParseErrors --hints:off {file}"
+    let (output, exitCode) = execCmdEx(cmd)
+    let expected = &"{file}(2, 4) Error: '***' expected\n{file}(2, 20) Error: '``' expected\n{file}(3, 32) Error: '```' expected\n"
+    checkrunner expected
+    check fileExists("minvalidrst.html")