Implement Pandoc Markdown concise link extension (#20304)
* Implement Pandoc Markdown concise link extension This implements https://github.com/nim-lang/Nim/issues/20127. Besides reference to headings we also support doing references to Nim symbols inside Nim modules. Markdown: ``` Some heading ------------ Ref. [Some heading]. ``` Nim: ``` proc someFunction*() ... ... ## Ref. [someFunction] ``` This is substitution for RST syntax like `` `target`_ ``. All 3 syntax variants of extension from Pandoc Markdown are supported: `[target]`, `[target][]`, `[description][target]`. This PR also fixes clashes in existing files, particularly conflicts with RST footnote feature, which does not work with this PR (but there is a plan to adopt a popular [Markdown footnote extension](https://pandoc.org/MANUAL.html#footnotes) to make footnotes work). Also the PR fixes a bug that Markdown links did not work when `[...]` section had a line break. The implementation is straightforward since link resolution did not change w.r.t. RST implementation, it's almost only about new syntax addition. The only essential difference is a possibility to add a custom link description: form `[description][target]` which does not have an RST equivalent. * fix nim 1.0 gotcha
This commit is contained in:
Andrey Makarov
GitHub
parent
b931e74a59
commit
cde6b2aab8
23 changed files with 325 additions and 152 deletions
|
|
@ -10,74 +10,101 @@ discard """
|
|||
import ../../lib/packages/docutils/[rstast, rst, dochelpers]
|
||||
import unittest
|
||||
|
||||
proc rstParseTest(text: string): PRstNode =
|
||||
proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind,
|
||||
arg: string) =
|
||||
doAssert msgkind == mwBrokenLink
|
||||
proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind,
|
||||
arg: string) =
|
||||
doAssert msgkind == mwBrokenLink
|
||||
|
||||
proc fromRst(text: string): LangSymbol =
|
||||
let r = rstParse(text, "-input-", LineRstInit, ColRstInit,
|
||||
{roNimFile},
|
||||
msgHandler=testMsgHandler)
|
||||
assert r.node.kind == rnRstRef
|
||||
result = toLangSymbol(r.node)
|
||||
|
||||
proc fromMd(text: string): LangSymbol =
|
||||
let r = rstParse(text, "-input-", LineRstInit, ColRstInit,
|
||||
{roPreferMarkdown, roSupportMarkdown, roNimFile},
|
||||
msgHandler=testMsgHandler)
|
||||
result = r.node
|
||||
assert r.node.kind == rnPandocRef
|
||||
assert r.node.len == 2
|
||||
# this son is the target:
|
||||
assert r.node.sons[1].kind == rnInner
|
||||
result = toLangSymbol(r.node.sons[1])
|
||||
|
||||
suite "Integration with Nim":
|
||||
test "simple symbol parsing (shortest form)":
|
||||
let input1 = "g_".rstParseTest
|
||||
check input1.toLangSymbol == LangSymbol(symKind: "", name: "g")
|
||||
let expected = LangSymbol(symKind: "", name: "g")
|
||||
check "g_".fromRst == expected
|
||||
check "[g]".fromMd == expected
|
||||
# test also alternative syntax variants of Pandoc Markdown:
|
||||
check "[g][]".fromMd == expected
|
||||
check "[this symbol][g]".fromMd == expected
|
||||
|
||||
test "simple symbol parsing (group of words)":
|
||||
let input1 = "`Y`_".rstParseTest
|
||||
check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y")
|
||||
#let input1 = "`Y`_".rstParseTest
|
||||
let expected1 = LangSymbol(symKind: "", name: "Y")
|
||||
check "`Y`_".fromRst == expected1
|
||||
check "[Y]".fromMd == expected1
|
||||
|
||||
# this means not a statement 'type', it's a backticked identifier `type`:
|
||||
let input2 = "`type`_".rstParseTest
|
||||
check input2.toLangSymbol == LangSymbol(symKind: "", name: "type")
|
||||
let expected2 = LangSymbol(symKind: "", name: "type")
|
||||
check "`type`_".fromRst == expected2
|
||||
check "[type]".fromMd == expected2
|
||||
|
||||
let input3 = "`[]`_".rstParseTest
|
||||
check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]")
|
||||
let expected3 = LangSymbol(symKind: "", name: "[]")
|
||||
check "`[]`_".fromRst == expected3
|
||||
# Markdown syntax for this case is NOT [[]]
|
||||
check "[`[]`]".fromMd == expected3
|
||||
|
||||
let input4 = "`X Y Z`_".rstParseTest
|
||||
check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz")
|
||||
let expected4 = LangSymbol(symKind: "", name: "Xyz")
|
||||
check "`X Y Z`_".fromRst == expected4
|
||||
check "[X Y Z]".fromMd == expected4
|
||||
|
||||
test "simple proc parsing":
|
||||
let input1 = "proc f".rstParseTest
|
||||
check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f")
|
||||
let expected = LangSymbol(symKind: "proc", name: "f")
|
||||
check "`proc f`_".fromRst == expected
|
||||
check "[proc f]".fromMd == expected
|
||||
|
||||
test "another backticked name":
|
||||
let input1 = """`template \`type\``_""".rstParseTest
|
||||
check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type")
|
||||
let expected = LangSymbol(symKind: "template", name: "type")
|
||||
check """`template \`type\``_""".fromRst == expected
|
||||
# no backslash in Markdown:
|
||||
check """[template `type`]""".fromMd == expected
|
||||
|
||||
test "simple proc parsing with parameters":
|
||||
let input1 = "`proc f*()`_".rstParseTest
|
||||
let input2 = "`proc f()`_".rstParseTest
|
||||
let expected = LangSymbol(symKind: "proc", name: "f",
|
||||
parametersProvided: true)
|
||||
check input1.toLangSymbol == expected
|
||||
check input2.toLangSymbol == expected
|
||||
check "`proc f*()`_".fromRst == expected
|
||||
check "`proc f()`_".fromRst == expected
|
||||
check "[proc f*()]".fromMd == expected
|
||||
check "[proc f()]".fromMd == expected
|
||||
|
||||
test "symbol parsing with 1 parameter":
|
||||
let input = "`f(G[int])`_".rstParseTest
|
||||
let expected = LangSymbol(symKind: "", name: "f",
|
||||
parameters: @[("G[int]", "")],
|
||||
parametersProvided: true)
|
||||
check input.toLangSymbol == expected
|
||||
check "`f(G[int])`_".fromRst == expected
|
||||
check "[f(G[int])]".fromMd == expected
|
||||
|
||||
test "more proc parsing":
|
||||
let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest
|
||||
let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest
|
||||
let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest
|
||||
let input1 = "`proc f[T](x:G[T]):M[T]`_".fromRst
|
||||
let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".fromRst
|
||||
let input3 = "`proc f*[T](x: G[T]): M[T]`_".fromRst
|
||||
let expected = LangSymbol(symKind: "proc",
|
||||
name: "f",
|
||||
generics: "[T]",
|
||||
parameters: @[("x", "G[T]")],
|
||||
parametersProvided: true,
|
||||
outType: "M[T]")
|
||||
check(input1.toLangSymbol == expected)
|
||||
check(input2.toLangSymbol == expected)
|
||||
check(input3.toLangSymbol == expected)
|
||||
check(input1 == expected)
|
||||
check(input2 == expected)
|
||||
check(input3 == expected)
|
||||
|
||||
test "advanced proc parsing with Nim identifier normalization":
|
||||
let input = """`proc binarySearch*[T, K](a: openarray[T]; key: K;
|
||||
cmp: proc (x: T; y: K): int)`_""".rstParseTest
|
||||
let inputRst = """`proc binarySearch*[T, K](a: openarray[T]; key: K;
|
||||
cmp: proc (x: T; y: K): int)`_"""
|
||||
let inputMd = """[proc binarySearch*[T, K](a: openarray[T]; key: K;
|
||||
cmp: proc (x: T; y: K): int)]"""
|
||||
let expected = LangSymbol(symKind: "proc",
|
||||
name: "binarysearch",
|
||||
generics: "[T,K]",
|
||||
|
|
@ -87,11 +114,12 @@ suite "Integration with Nim":
|
|||
("cmp", "proc(x:T;y:K):int")],
|
||||
parametersProvided: true,
|
||||
outType: "")
|
||||
check(input.toLangSymbol == expected)
|
||||
check(inputRst.fromRst == expected)
|
||||
check(inputMd.fromMd == expected)
|
||||
|
||||
test "the same without proc":
|
||||
let input = """`binarySearch*[T, K](a: openarray[T]; key: K;
|
||||
cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest
|
||||
cmp: proc (x: T; y: K): int {.closure.})`_"""
|
||||
let expected = LangSymbol(symKind: "",
|
||||
name: "binarysearch",
|
||||
generics: "[T,K]",
|
||||
|
|
@ -101,27 +129,32 @@ suite "Integration with Nim":
|
|||
("cmp", "proc(x:T;y:K):int")],
|
||||
parametersProvided: true,
|
||||
outType: "")
|
||||
check(input.toLangSymbol == expected)
|
||||
check(input.fromRst == expected)
|
||||
let inputMd = """[binarySearch*[T, K](a: openarray[T]; key: K;
|
||||
cmp: proc (x: T; y: K): int {.closure.})]"""
|
||||
check(inputMd.fromMd == expected)
|
||||
|
||||
test "operator $ with and without backticks":
|
||||
let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""".
|
||||
rstParseTest
|
||||
let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""".
|
||||
rstParseTest
|
||||
let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_"""
|
||||
let input1md = "[func `$`*[T](a: `open Array`[T]): string]"
|
||||
let input2 = """`func $*[T](a: \`open Array\`[T]): string`_"""
|
||||
let input2md = "[func $*[T](a: `open Array`[T]): string]"
|
||||
let expected = LangSymbol(symKind: "func",
|
||||
name: "$",
|
||||
generics: "[T]",
|
||||
parameters: @[("a", "openarray[T]")],
|
||||
parametersProvided: true,
|
||||
outType: "string")
|
||||
check(input1.toLangSymbol == expected)
|
||||
check(input2.toLangSymbol == expected)
|
||||
check input1.fromRst == expected
|
||||
check input2.fromRst == expected
|
||||
check input1md.fromMd == expected
|
||||
check input2md.fromMd == expected
|
||||
|
||||
test "operator [] with and without backticks":
|
||||
let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""".
|
||||
rstParseTest
|
||||
let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""".
|
||||
rstParseTest
|
||||
let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_"""
|
||||
let input1md = "[func `[]`[T](a: `open Array`[T], idx: int): T]"
|
||||
let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_"""
|
||||
let input2md = "[func [][T](a: `open Array`[T], idx: int): T]"
|
||||
let expected = LangSymbol(symKind: "func",
|
||||
name: "[]",
|
||||
generics: "[T]",
|
||||
|
|
@ -129,21 +162,25 @@ suite "Integration with Nim":
|
|||
("idx", "int")],
|
||||
parametersProvided: true,
|
||||
outType: "T")
|
||||
check(input1.toLangSymbol == expected)
|
||||
check(input2.toLangSymbol == expected)
|
||||
check input1.fromRst == expected
|
||||
check input2.fromRst == expected
|
||||
check input1md.fromMd == expected
|
||||
check input2md.fromMd == expected
|
||||
|
||||
test "postfix symbol specifier #1":
|
||||
let input = """`walkDir iterator`_""".
|
||||
rstParseTest
|
||||
let input = "`walkDir iterator`_"
|
||||
let inputMd = "[walkDir iterator]"
|
||||
let expected = LangSymbol(symKind: "iterator",
|
||||
name: "walkdir")
|
||||
check(input.toLangSymbol == expected)
|
||||
check input.fromRst == expected
|
||||
check inputMd.fromMd == expected
|
||||
|
||||
test "postfix symbol specifier #2":
|
||||
let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""".
|
||||
rstParseTest
|
||||
let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""".
|
||||
rstParseTest
|
||||
let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_"""
|
||||
let input1md = "[`[]`[T](a: `open Array`[T], idx: int): T func]"
|
||||
let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_"""
|
||||
# note again that ` is needed between 1st and second [
|
||||
let input2md = "[`[]`[T](a: `open Array`[T], idx: int): T func]"
|
||||
let expected = LangSymbol(symKind: "func",
|
||||
name: "[]",
|
||||
generics: "[T]",
|
||||
|
|
@ -151,11 +188,16 @@ suite "Integration with Nim":
|
|||
("idx", "int")],
|
||||
parametersProvided: true,
|
||||
outType: "T")
|
||||
check(input1.toLangSymbol == expected)
|
||||
check(input2.toLangSymbol == expected)
|
||||
check input1.fromRst == expected
|
||||
check input2.fromRst == expected
|
||||
check input1md.fromMd == expected
|
||||
check input2md.fromMd == expected
|
||||
|
||||
test "type of type":
|
||||
check ("`CopyFlag enum`_".rstParseTest.toLangSymbol ==
|
||||
LangSymbol(symKind: "type",
|
||||
symTypeKind: "enum",
|
||||
name: "Copyflag"))
|
||||
let inputRst = "`CopyFlag enum`_"
|
||||
let inputMd = "[CopyFlag enum]"
|
||||
let expected = LangSymbol(symKind: "type",
|
||||
symTypeKind: "enum",
|
||||
name: "Copyflag")
|
||||
check inputRst.fromRst == expected
|
||||
check inputMd.fromMd == expected
|
||||
|
|
|
|||
|
|
@ -1173,7 +1173,7 @@ suite "Warnings":
|
|||
lastParagraph
|
||||
"""
|
||||
var warnings = new seq[string]
|
||||
let output = input.toAst(warnings=warnings)
|
||||
let output = input.toAst(rstOptions=preferRst, warnings=warnings)
|
||||
check(warnings[] == @[
|
||||
"input(3, 14) Warning: broken link 'citation-som'",
|
||||
"input(5, 7) Warning: broken link 'a broken Link'",
|
||||
|
|
@ -1199,7 +1199,7 @@ suite "Warnings":
|
|||
rnParagraph
|
||||
rnLeaf 'here'
|
||||
rnLeaf ' '
|
||||
rnRef
|
||||
rnRstRef
|
||||
rnLeaf 'brokenLink'
|
||||
""")
|
||||
removeFile("other.rst")
|
||||
|
|
@ -1558,7 +1558,7 @@ suite "RST inline markup":
|
|||
|
||||
test "no punctuation in the end of a standalone URI is allowed":
|
||||
check(dedent"""
|
||||
[see (http://no.org)], end""".toAst ==
|
||||
[see (http://no.org)], end""".toAst(rstOptions = preferRst) ==
|
||||
dedent"""
|
||||
rnInner
|
||||
rnLeaf '['
|
||||
|
|
@ -1606,6 +1606,19 @@ suite "RST inline markup":
|
|||
rnLeaf 'end'
|
||||
""")
|
||||
|
||||
test "Markdown-style link can be split to a few lines":
|
||||
check(dedent"""
|
||||
is [term-rewriting
|
||||
macros](manual.html#term-rewriting-macros)""".toAst ==
|
||||
dedent"""
|
||||
rnInner
|
||||
rnLeaf 'is'
|
||||
rnLeaf ' '
|
||||
rnHyperlink
|
||||
rnLeaf 'term-rewriting macros'
|
||||
rnLeaf 'manual.html#term-rewriting-macros'
|
||||
""")
|
||||
|
||||
test "URL with balanced parentheses (Markdown rule)":
|
||||
# 2 balanced parens, 1 unbalanced:
|
||||
check(dedent"""
|
||||
|
|
|
|||
|
|
@ -983,7 +983,7 @@ Test1
|
|||
|
||||
Ref. [#note]_
|
||||
"""
|
||||
let output1 = input1.toHtml
|
||||
let output1 = input1.toHtml(preferRst)
|
||||
doAssert output1.count(">[1]</a>") == 1
|
||||
doAssert output1.count(">[2]</a>") == 2
|
||||
doAssert "href=\"#footnote-note\"" in output1
|
||||
|
|
@ -1001,7 +1001,7 @@ Test1
|
|||
|
||||
Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_.
|
||||
"""
|
||||
let output2 = input2.toHtml
|
||||
let output2 = input2.toHtml(preferRst)
|
||||
doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_."
|
||||
|
||||
# check that auto-symbol footnotes work:
|
||||
|
|
@ -1017,7 +1017,7 @@ Test1
|
|||
|
||||
And [*]_.
|
||||
"""
|
||||
let output3 = input3.toHtml
|
||||
let output3 = input3.toHtml(preferRst)
|
||||
# both references and footnotes. Footnotes have link to themselves.
|
||||
doAssert output3.count("href=\"#footnotesym-1\">[*]</a>") == 2
|
||||
doAssert output3.count("href=\"#footnotesym-2\">[**]</a>") == 2
|
||||
|
|
@ -1047,7 +1047,7 @@ Test1
|
|||
|
||||
Ref. [#note]_ and [#]_ and [#]_.
|
||||
"""
|
||||
let output4 = input4.toHtml
|
||||
let output4 = input4.toHtml(preferRst)
|
||||
doAssert ">[-1]" notin output1
|
||||
let order = @[
|
||||
"footnote-3", "[3]", "Manual1.",
|
||||
|
|
@ -1072,7 +1072,7 @@ Test1
|
|||
Ref. [#note]_
|
||||
"""
|
||||
var error5 = new string
|
||||
let output5 = input5.toHtml(error=error5)
|
||||
let output5 = input5.toHtml(preferRst, error=error5)
|
||||
check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " &
|
||||
"and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " &
|
||||
"footnotes")
|
||||
|
|
@ -1086,7 +1086,7 @@ Test1
|
|||
Ref. [*]_
|
||||
"""
|
||||
var error6 = new string
|
||||
let output6 = input6.toHtml(error=error6)
|
||||
let output6 = input6.toHtml(preferRst, error=error6)
|
||||
check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " &
|
||||
"and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " &
|
||||
"footnotes")
|
||||
|
|
@ -1096,7 +1096,7 @@ Test1
|
|||
|
||||
Ref. [some:citation-2020]_.
|
||||
"""
|
||||
let output7 = input7.toHtml
|
||||
let output7 = input7.toHtml(preferRst)
|
||||
doAssert output7.count("href=\"#citation-somecoloncitationminus2020\"") == 2
|
||||
doAssert output7.count("[Some:CITATION-2020]") == 1
|
||||
doAssert output7.count("[some:citation-2020]") == 1
|
||||
|
|
@ -1109,7 +1109,7 @@ Test1
|
|||
Ref. [som]_.
|
||||
"""
|
||||
var warnings8 = new seq[string]
|
||||
let output8 = input8.toHtml(warnings=warnings8)
|
||||
let output8 = input8.toHtml(preferRst, warnings=warnings8)
|
||||
check(warnings8[] == @["input(3, 7) Warning: broken link 'citation-som'"])
|
||||
|
||||
# check that footnote group does not break parsing of other directives:
|
||||
|
|
@ -1145,7 +1145,7 @@ Test1
|
|||
|
||||
.. [Third] Citation.
|
||||
"""
|
||||
let output10 = input10.toHtml
|
||||
let output10 = input10.toHtml(preferRst)
|
||||
doAssert output10.count("<hr class=\"footnote\">" &
|
||||
"<div class=\"footnote-group\">") == 3
|
||||
doAssert output10.count("<div class=\"footnote-label\">") == 3
|
||||
|
|
@ -1165,7 +1165,7 @@ Test1
|
|||
.. [#] Body3
|
||||
.. [2] Body2.
|
||||
"""
|
||||
let output12 = input12.toHtml
|
||||
let output12 = input12.toHtml(preferRst)
|
||||
let orderAuto = @[
|
||||
"#footnoteauto-1", "[1]",
|
||||
"#footnoteauto-2", "[3]",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue