Markdown code blocks part 2; migrate Nim Manual (#20080)

* Change headings underscored by `~~~` to `###`

* Markdown code blocks part 2; migrate Nim Manual

doc/backends.md

@@ -155,8 +155,7 @@ To wrap native code, take a look at the `c2nim tool <https://github.com/nim-lang
 with the process of scanning and transforming header files into a Nim
 interface.
 
-C invocation example
+### C invocation example
-~~~~~~~~~~~~~~~~~~~~
 
 Create a ``logic.c`` file with the following content:
 
@@ -194,8 +193,7 @@ could as well pass ``logic.o``) we could be passing switches to link any other
 static C library.
 
 
-JavaScript invocation example
+### JavaScript invocation example
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Create a ``host.html`` file with the following content:
 
@@ -251,8 +249,7 @@ The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch.
 Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`.
 
 
-Nim invocation example from C
+### Nim invocation example from C
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Create a ``fib.nim`` file with the following content:
 
@@ -312,8 +309,7 @@ vary for each system. For instance, on Linux systems you will likely need to
 use `-ldl`:option: too to link in required dlopen functionality.
 
 
-Nim invocation example from JavaScript
+### Nim invocation example from JavaScript
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Create a ``mhost.html`` file with the following content:
 

doc/manual_experimental.md

@@ -1463,8 +1463,7 @@ The operators `*`,  `**`, `|`, `~` have a special meaning in patterns
 if they are written in infix notation.
 
 
-The `|` operator
+### The `|` operator
-~~~~~~~~~~~~~~~~~~
 
 The `|` operator if used as infix operator creates an ordered choice:
 
@@ -1490,8 +1489,7 @@ semantics anyway. In fact, they can be deactivated with the `--patterns:off`:opt
 command line option or temporarily with the `patterns` pragma.
 
 
-The `{}` operator
+### The `{}` operator
-~~~~~~~~~~~~~~~~~~~
 
 A pattern expression can be bound to a pattern parameter via the `expr{param}`
 notation:
@@ -1504,8 +1502,7 @@ notation:
   echo a
 
 
-The `~` operator
+### The `~` operator
-~~~~~~~~~~~~~~~~~~
 
 The `~` operator is the 'not' operator in patterns:
 
@@ -1523,8 +1520,7 @@ The `~` operator is the 'not' operator in patterns:
   echo a
 
 
-The `*` operator
+### The `*` operator
-~~~~~~~~~~~~~~~~~~
 
 The `*` operator can *flatten* a nested binary expression like `a & b & c`
 to `&(a, b, c)`:
@@ -1559,8 +1555,7 @@ produces:
    `&&`("my", space & "awe", "some ", "concat")
 
 
-The `**` operator
+### The `**` operator
-~~~~~~~~~~~~~~~~~~~
 
 The `**` is much like the `*` operator, except that it gathers not only
 all the arguments, but also the matched operators in reverse polish notation:

doc/sets_fragment.txt

@@ -52,8 +52,7 @@ operation             meaning
 `excl(A, elem)`       same as `A = A - {elem}`
 ==================    ========================================================
 
-Bit fields
+### Bit fields
-~~~~~~~~~~
 
 Sets are often used to define a type for the *flags* of a procedure.
 This is a cleaner (and type safe) solution than defining integer

lib/packages/docutils/rst.nim

@@ -11,9 +11,9 @@
 ##       packages/docutils/rst
 ## ==================================
 ##
-## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## ------------------------------------------
 ## Nim-flavored reStructuredText and Markdown
-## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## ------------------------------------------
 ##
 ## This module implements a `reStructuredText`:idx: (RST) and
 ## `Markdown`:idx: parser.
@@ -290,7 +290,7 @@ type
 
 proc rstnodeToRefname*(n: PRstNode): string
 proc addNodes*(n: PRstNode): string
-proc getFieldValue*(n: PRstNode, fieldname: string): string
+proc getFieldValue*(n: PRstNode, fieldname: string): string {.gcsafe.}
 proc getArgument*(n: PRstNode): string
 
 # ----------------------------- scanner part --------------------------------
@@ -1675,19 +1675,33 @@ proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode =
       field.add(fieldBody)
       result.add(field)
 
+proc mayLoadFile(p: RstParser, result: var PRstNode) =
+  var filename = strip(getFieldValue(result, "file"),
+                       chars = Whitespace + {'"'})
+  if filename != "":
+    if roSandboxDisabled notin p.s.options:
+      let tok = p.tok[p.idx-2]
+      rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col)
+    var path = p.findRelativeFile(filename)
+    if path == "": rstMessage(p, meCannotOpenFile, filename)
+    var n = newRstNode(rnLiteralBlock)
+    n.add newLeaf(readFile(path))
+    result.sons[2] = n
+
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
   result = newRstNodeA(p, rnCodeBlock)
+  result.sons.setLen(3)
   let line = curLine(p)
   let baseCol = currentTok(p).col
   let baseSym = currentTok(p).symbol  # usually just ```
   inc p.idx
   result.info = lineInfo(p)
   var args = newRstNode(rnDirArg)
-  var fields: PRstNode = nil
   if currentTok(p).kind == tkWord:
     args.add(newLeaf(p))
     inc p.idx
-    fields = parseMarkdownCodeblockFields(p)
+    result.sons[1] = parseMarkdownCodeblockFields(p)
+    mayLoadFile(p, result)
   else:
     args = nil
   var n = newLeaf("")
@@ -1712,11 +1726,11 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
     else:
       n.text.add(currentTok(p).symbol)
       inc p.idx
-  var lb = newRstNode(rnLiteralBlock)
+  result.sons[0] = args
-  lb.add(n)
+  if result.sons[2] == nil:
-  result.add(args)
+    var lb = newRstNode(rnLiteralBlock)
-  result.add(fields)
+    lb.add(n)
-  result.add(lb)
+    result.sons[2] = lb
 
 proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool =
   var desc, link = ""
@@ -1802,9 +1816,12 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
   p.idx = i
 
 proc isMarkdownCodeBlock(p: RstParser): bool =
+  template allowedSymbol: bool =
+    (currentTok(p).symbol[0] == '`' or
+      roPreferMarkdown in p.s.options and currentTok(p).symbol[0] == '~')
   result = (roSupportMarkdown in p.s.options and
             currentTok(p).kind in {tkPunct, tkAdornment} and
-            currentTok(p).symbol[0] == '`' and  # tilde ~ is not supported
+            allowedSymbol and
             currentTok(p).symbol.len >= 3)
 
 proc parseInline(p: var RstParser, father: PRstNode) =
@@ -2580,9 +2597,7 @@ proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int =
   if p.tok[result].kind == tkIndent: inc result
 
 proc checkColumns(p: RstParser, cols: RstCols) =
-  var
+  var i = p.idx
-    i = p.idx
-    col = 0
   if p.tok[i].symbol[0] != '=':
     rstMessage(p, mwRstStyle,
                "only tables with `=` columns specification are allowed")
@@ -3208,16 +3223,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ## file. This behaviour is disabled in sandboxed mode and can be re-enabled
   ## with the `roSandboxDisabled` flag.
   result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock)
-  var filename = strip(getFieldValue(result, "file"))
+  mayLoadFile(p, result)
-  if filename != "":
-    if roSandboxDisabled notin p.s.options:
-      let tok = p.tok[p.idx-2]
-      rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col)
-    var path = p.findRelativeFile(filename)
-    if path == "": rstMessage(p, meCannotOpenFile, filename)
-    var n = newRstNode(rnLiteralBlock)
-    n.add newLeaf(readFile(path))
-    result.sons[2] = n
 
   # Extend the field block if we are using our custom Nim extension.
   if nimExtension:

tests/stdlib/trst.nim

@@ -471,21 +471,34 @@ suite "RST parsing":
                   rnLeaf  '.'
           """)
 
+  let expectCodeBlock = dedent"""
+      rnCodeBlock
+        [nil]
+        [nil]
+        rnLiteralBlock
+          rnLeaf  '
+      let a = 1
+      ```'
+      """
+
   test "Markdown code blocks with more > 3 backticks":
     check(dedent"""
         ````
         let a = 1
         ```
-        ````""".toAst ==
+        ````""".toAst == expectCodeBlock)
-      dedent"""
+
-        rnCodeBlock
+  test "Markdown code blocks with ~~~":
-          [nil]
+    check(dedent"""
-          [nil]
+        ~~~
-          rnLiteralBlock
-            rnLeaf  '
         let a = 1
-        ```'
+        ```
-      """)
+        ~~~""".toAst == expectCodeBlock)
+    check(dedent"""
+        ~~~~~
+        let a = 1
+        ```
+        ~~~~~""".toAst == expectCodeBlock)
 
   test "Markdown code blocks with Nim-specific arguments":
     check(dedent"""

tests/stdlib/trstgen.nim

@@ -552,7 +552,7 @@ context1
 
 context2
 """
-    let output1 = input1.toHtml
+    let output1 = input1.toHtml(preferRst)
     doAssert "<hr" in output1
 
     let input2 = """
@@ -1616,3 +1616,8 @@ suite "local file inclusion":
     discard ".. code-block:: nim\n    :file: ./readme.md".toHtml(error=error)
     check(error[] == "input(2, 20) Error: disabled directive: 'file'")
 
+  test "code-block file directive is disabled - Markdown":
+    var error = new string
+    discard "```nim file = ./readme.md\n```".toHtml(error=error)
+    check(error[] == "input(1, 23) Error: disabled directive: 'file'")
+