Improve Markdown code blocks & start moving docs to Markdown style (#19954)

- add additional parameters parsing (other implementations will just
  ignore them). E.g. if in RST we have:

  .. code:: nim
     :test: "nim c $1"

     ...

  then in Markdown that will be:

  ```nim test="nim c $1"
  ...
  ```

- implement Markdown interpretation of additional indentation which is
  less than 4 spaces (>=4 spaces is a code block but it's not
implemented yet). RST interpretes it as quoted block, for Markdown it's
just normal paragraphs.
- add separate `md2html` and `md2tex` commands. This is to separate
  Markdown behavior in cases when it diverges w.r.t. RST significantly —
most conspicously like in the case of additional indentation above, and
also currently the contradicting inline rule of Markdown is also turned
on only in `md2html` and `md2tex`. **Rationale:** mixing Markdown and
RST arbitrarily is a way to nowhere, we need to provide a way to fix the
particular behavior. Note that still all commands have **both** Markdown
and RST features **enabled**. In this PR `*.nim` files can be processed
only in Markdown mode, while `md2html` is for `*.md` files and
`rst2html` for `*.rst` files.
- rename `*.rst` files to `.*md` as our current default behavior is
  already Markdown-ish
- convert code blocks in `docgen.rst` to Markdown style as an example.
  Other code blocks will be converted in the follow-up PRs
- fix indentation inside Markdown code blocks — additional indentation
  is preserved there
- allow more than 3 backticks open/close blocks (tildas \~ are still not
  allowed to avoid conflict with RST adornment headings) see also
https://github.com/nim-lang/RFCs/issues/355
- better error messages
- (other) fix a bug that admonitions cannot be used in sandbox mode; fix
  annoying warning on line 2711

compiler/docgen.nim

@@ -88,7 +88,7 @@ type
     jEntriesFinal: JsonNode    # final JSON after RST pass 2 and rendering
     types: TStrTable
     sharedState: PRstSharedState
-    isPureRst: bool
+    standaloneDoc: bool
     conf*: ConfigRef
     cache*: IdentCache
     exampleCounter: int
@@ -230,6 +230,7 @@ template declareClosures =
     case msgKind
     of meCannotOpenFile: k = errCannotOpenFile
     of meExpected: k = errXExpected
+    of meMissingClosing: k = errRstMissingClosing
     of meGridTableNotImplemented: k = errRstGridTableNotImplemented
     of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
     of meIllformedTable: k = errRstIllformedTable
@@ -276,16 +277,18 @@ proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
 
 proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
                     outExt: string = HtmlExt, module: PSym = nil,
-                    isPureRst = false): PDoc =
+                    standaloneDoc = false, preferMarkdown = true): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  result.isPureRst = isPureRst
-  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled}
-  if not isPureRst: options.incl roNimFile
+  result.standaloneDoc = standaloneDoc
+  var options= {roSupportRawDirective, roSupportMarkdown, roSandboxDisabled}
+  if preferMarkdown:
+    options.incl roPreferMarkdown
+  if not standaloneDoc: options.incl roNimFile
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
@@ -333,7 +336,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       # Make sure the destination directory exists
       createDir(outp.splitFile.dir)
       # Include the current file if we're parsing a nim file
-      let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
+      let importStmt = if d.standaloneDoc: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
       writeFile(outp, importStmt & content)
 
       proc interpSnippetCmd(cmd: string): string =
@@ -1512,7 +1515,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
+  let bodyname = if d.hasToc and not d.standaloneDoc and not d.conf.isLatexCmd:
                    groupsection.setLen 0
                    "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
@@ -1626,9 +1629,11 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
   generateIndex(d)
 
 proc commandRstAux(cache: IdentCache, conf: ConfigRef;
-                   filename: AbsoluteFile, outExt: string) =
+                   filename: AbsoluteFile, outExt: string,
+                   preferMarkdown: bool) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
+  var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true,
+                        preferMarkdown = preferMarkdown)
   let rst = parseRst(readFile(filen.string),
                      line=LineRstInit, column=ColRstInit,
                      conf, d.sharedState)
@@ -1637,11 +1642,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
   writeOutput(d)
   generateIndex(d)
 
-proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, HtmlExt)
+proc commandRst2Html*(cache: IdentCache, conf: ConfigRef,
+                      preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, HtmlExt, preferMarkdown)
 
-proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) =
-  commandRstAux(cache, conf, conf.projectFull, TexExt)
+proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef,
+                     preferMarkdown=false) =
+  commandRstAux(cache, conf, conf.projectFull, TexExt, preferMarkdown)
 
 proc commandJson*(cache: IdentCache, conf: ConfigRef) =
   ## implementation of a deprecated jsondoc0 command