Improve Markdown code blocks & start moving docs to Markdown style (#19954)

- add additional parameters parsing (other implementations will just
  ignore them). E.g. if in RST we have:

  .. code:: nim
     :test: "nim c $1"

     ...

  then in Markdown that will be:

  ```nim test="nim c $1"
  ...
  ```

- implement Markdown interpretation of additional indentation which is
  less than 4 spaces (>=4 spaces is a code block but it's not
implemented yet). RST interpretes it as quoted block, for Markdown it's
just normal paragraphs.
- add separate `md2html` and `md2tex` commands. This is to separate
  Markdown behavior in cases when it diverges w.r.t. RST significantly —
most conspicously like in the case of additional indentation above, and
also currently the contradicting inline rule of Markdown is also turned
on only in `md2html` and `md2tex`. **Rationale:** mixing Markdown and
RST arbitrarily is a way to nowhere, we need to provide a way to fix the
particular behavior. Note that still all commands have **both** Markdown
and RST features **enabled**. In this PR `*.nim` files can be processed
only in Markdown mode, while `md2html` is for `*.md` files and
`rst2html` for `*.rst` files.
- rename `*.rst` files to `.*md` as our current default behavior is
  already Markdown-ish
- convert code blocks in `docgen.rst` to Markdown style as an example.
  Other code blocks will be converted in the follow-up PRs
- fix indentation inside Markdown code blocks — additional indentation
  is preserved there
- allow more than 3 backticks open/close blocks (tildas \~ are still not
  allowed to avoid conflict with RST adornment headings) see also
https://github.com/nim-lang/RFCs/issues/355
- better error messages
- (other) fix a bug that admonitions cannot be used in sandbox mode; fix
  annoying warning on line 2711

tests/stdlib/trst.nim

@@ -24,8 +24,11 @@ import unittest, strutils
 import std/private/miscdollars
 import os
 
+const preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}
+const preferRst = {roSupportMarkdown, roNimFile, roSandboxDisabled}
+
 proc toAst(input: string,
-            rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled},
+            rstOptions: RstParseOptions = preferMarkdown,
             error: ref string = nil,
             warnings: ref seq[string] = nil): string =
   ## If `error` is nil then no errors should be generated.
@@ -451,7 +454,7 @@ suite "RST parsing":
         >   - y
         >
         > Paragraph.
-        """.toAst == dedent"""
+        """.toAst(rstOptions = preferRst) == dedent"""
           rnMarkdownBlockQuote
             rnMarkdownBlockQuoteItem  quotationDepth=1
               rnInner
@@ -468,6 +471,93 @@ suite "RST parsing":
                   rnLeaf  '.'
           """)
 
+  test "Markdown code blocks with more > 3 backticks":
+    check(dedent"""
+        ````
+        let a = 1
+        ```
+        ````""".toAst ==
+      dedent"""
+        rnCodeBlock
+          [nil]
+          [nil]
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1
+        ```'
+      """)
+
+  test "Markdown code blocks with Nim-specific arguments":
+    check(dedent"""
+        ```nim number-lines=1 test
+        let a = 1
+        ```""".toAst ==
+      dedent"""
+        rnCodeBlock
+          rnDirArg
+            rnLeaf  'nim'
+          rnFieldList
+            rnField
+              rnFieldName
+                rnLeaf  'number-lines'
+              rnFieldBody
+                rnLeaf  '1'
+            rnField
+              rnFieldName
+                rnLeaf  'test'
+              rnFieldBody
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1'
+        """)
+
+    check(dedent"""
+        ```nim test = "nim c $1"  number-lines = 1
+        let a = 1
+        ```""".toAst ==
+      dedent"""
+        rnCodeBlock
+          rnDirArg
+            rnLeaf  'nim'
+          rnFieldList
+            rnField
+              rnFieldName
+                rnLeaf  'test'
+              rnFieldBody
+                rnLeaf  '"nim c $1"'
+            rnField
+              rnFieldName
+                rnLeaf  'number-lines'
+              rnFieldBody
+                rnLeaf  '1'
+          rnLiteralBlock
+            rnLeaf  '
+        let a = 1'
+        """)
+
+  test "additional indentation < 4 spaces is handled fine":
+    check(dedent"""
+        Indentation
+
+          ```nim
+            let a = 1
+          ```""".toAst ==
+      dedent"""
+        rnInner
+          rnParagraph
+            rnLeaf  'Indentation'
+          rnParagraph
+            rnCodeBlock
+              rnDirArg
+                rnLeaf  'nim'
+              [nil]
+              rnLiteralBlock
+                rnLeaf  '
+          let a = 1'
+      """)
+      # | |
+      # |  \ indentation of exactly two spaces before 'let a = 1'
+
   test "option list has priority over definition list":
     check(dedent"""
         --defusages
@@ -562,7 +652,7 @@ suite "RST parsing":
 
          notAcomment1
          notAcomment2
-        someParagraph""".toAst ==
+        someParagraph""".toAst(rstOptions = preferRst) ==
       dedent"""
         rnInner
           rnBlockQuote
@@ -574,6 +664,25 @@ suite "RST parsing":
             rnLeaf  'someParagraph'
         """)
 
+  test "check that additional line right after .. ends comment (Markdown mode)":
+    # in Markdown small indentation does not matter so this should
+    # just be split to 2 paragraphs.
+    check(dedent"""
+        ..
+
+         notAcomment1
+         notAcomment2
+        someParagraph""".toAst ==
+      dedent"""
+        rnInner
+          rnInner
+            rnLeaf  'notAcomment1'
+            rnLeaf  ' '
+            rnLeaf  'notAcomment2'
+          rnParagraph
+            rnLeaf  'someParagraph'
+        """)
+
   test "but blank lines after 2nd non-empty line don't end the comment":
     check(dedent"""
         ..
@@ -592,7 +701,7 @@ suite "RST parsing":
 
         ..
 
-          someBlockQuote""".toAst ==
+          someBlockQuote""".toAst(rstOptions = preferRst) ==
       dedent"""
         rnInner
           rnAdmonition  adType=note