jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

further progress on rst roles & directives (fix #17646) (#17659)

Browse source 
* further progress on rst roles & dir-s (fix #17646)

* fix documents according to the messages

* fix bug 17 from #17340
This commit is contained in:
Andrey Makarov 2021-04-08 20:00:14 +03:00 • committed by GitHub
commit 42687457b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 129 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/packages/docutils/highlite.nim
View file

@ -13,7 +13,7 @@
##
## You can use this to build your own syntax highlighting, check this example:
##
## .. code::nim
## .. code:: Nim
## let code = """for x in $int.high: echo x.ord mod 2 == 0"""
## var toknizr: GeneralTokenizer
## initGeneralTokenizer(toknizr, code)
@ -34,7 +34,7 @@
##
## The proc `getSourceLanguage` can get the language `enum` from a string:
##
## .. code::nim
## .. code:: Nim
## for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)
##

110
lib/packages/docutils/rst.nim
View file

@ -912,6 +912,38 @@ template newLeaf(s: string): PRstNode = newRstLeaf(s)
proc newLeaf(p: var RstParser): PRstNode =
result = newLeaf(currentTok(p).symbol)
proc validRefnamePunct(x: string): bool =
## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
func getRefnameIdx(p: RstParser, startIdx: int): int =
## Gets last token index of a refname ("word" in RST terminology):
##
## reference names are single words consisting of alphanumerics plus
## isolated (no two adjacent) internal hyphens, underscores, periods,
## colons and plus signs; no whitespace or other characters are allowed.
##
## Refnames are used for:
## - reference names
## - role names
## - directive names
## - footnote labels
##
# TODO: use this func in all other relevant places
var j = startIdx
if p.tok[j].kind == tkWord:
inc j
while p.tok[j].kind == tkPunct and validRefnamePunct(p.tok[j].symbol) and
p.tok[j+1].kind == tkWord:
inc j, 2
result = j - 1
func getRefname(p: RstParser, startIdx: int): (string, int) =
let lastIdx = getRefnameIdx(p, startIdx)
result[1] = lastIdx
for j in startIdx..lastIdx:
result[0].add p.tok[j].symbol
proc getReferenceName(p: var RstParser, endStr: string): PRstNode =
var res = newRstNode(rnInner)
while true:
@ -1011,7 +1043,10 @@ proc match(p: RstParser, start: int, expr: string): bool =
var last = expr.len - 1
while i <= last:
case expr[i]
of 'w': result = p.tok[j].kind == tkWord
of 'w':
let lastIdx = getRefnameIdx(p, j)
result = lastIdx >= j
if result: j = lastIdx
of ' ': result = p.tok[j].kind == tkWhite
of 'i': result = p.tok[j].kind == tkIndent
of 'I': result = p.tok[j].kind in {tkIndent, tkEof}
@ -1058,7 +1093,7 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
proc whichRole(p: RstParser, sym: string): RstNodeKind =
result = whichRoleAux(sym)
if result == rnUnknownRole:
rstMessage(p, mwUnsupportedLanguage, p.s.currRole)
rstMessage(p, mwUnsupportedLanguage, sym)
proc toInlineCode(n: PRstNode, language: string): PRstNode =
## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
@ -1078,6 +1113,11 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode =
lb.add newLeaf(s)
result.add lb
proc toUnknownRole(n: PRstNode, roleName: string): PRstNode =
let newN = newRstNode(rnInner, n.sons)
let newSons = @[newN, newLeaf(roleName)]
result = newRstNode(rnUnknownRole, newSons)
proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
var newKind = n.kind
var newSons = n.sons
@ -1102,17 +1142,15 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
result = newRstNode(newKind, newSons)
elif match(p, p.idx, ":w:"):
# a role:
let roleName = nextTok(p).symbol
let (roleName, lastIdx) = getRefname(p, p.idx+1)
newKind = whichRole(p, roleName)
if newKind == rnUnknownRole:
let newN = newRstNode(rnInner, n.sons)
newSons = @[newN, newLeaf(roleName)]
result = newRstNode(newKind, newSons)
result = n.toUnknownRole(roleName)
elif newKind == rnInlineCode:
result = n.toInlineCode(language=roleName)
else:
result = newRstNode(newKind, newSons)
inc p.idx, 3
p.idx = lastIdx + 2
else:
if p.s.currRoleKind == rnInlineCode:
result = n.toInlineCode(language=p.s.currRole)
@ -1139,10 +1177,6 @@ proc parseSmiley(p: var RstParser): PRstNode =
result.text = val
return
proc validRefnamePunct(x: string): bool =
## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names
x.len == 1 and x[0] in {'-', '_', '.', ':', '+'}
proc isUrl(p: RstParser, i: int): bool =
result = p.tok[i+1].symbol == ":" and p.tok[i+2].symbol == "//" and
p.tok[i+3].kind == tkWord and
@ -1373,14 +1407,18 @@ proc parseInline(p: var RstParser, father: PRstNode) =
var n = newRstNode(rnInlineLiteral)
parseUntil(p, n, "``", false)
father.add(n)
elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
let roleName = nextTok(p).symbol
elif match(p, p.idx, ":w:") and
(var lastIdx = getRefnameIdx(p, p.idx + 1);
p.tok[lastIdx+2].symbol == "`"):
let (roleName, _) = getRefname(p, p.idx+1)
let k = whichRole(p, roleName)
var n = newRstNode(k)
inc p.idx, 3
p.idx = lastIdx + 2
if k == rnInlineCode:
n = n.toInlineCode(language=roleName)
parseUntil(p, n, "`", false) # bug #17260
if k == rnUnknownRole:
n = n.toUnknownRole(roleName)
father.add(n)
elif isInlineMarkupStart(p, "`"):
var n = newRstNode(rnInterpretedText)
@ -1438,25 +1476,28 @@ proc parseInline(p: var RstParser, father: PRstNode) =
else: discard
proc getDirective(p: var RstParser): string =
if currentTok(p).kind == tkWhite and nextTok(p).kind == tkWord:
var j = p.idx
inc p.idx
result = currentTok(p).symbol
inc p.idx
while currentTok(p).kind in {tkWord, tkPunct, tkAdornment, tkOther}:
if currentTok(p).symbol == "::": break
result.add(currentTok(p).symbol)
inc p.idx
if currentTok(p).kind == tkWhite: inc p.idx
if currentTok(p).symbol == "::":
inc p.idx
if currentTok(p).kind == tkWhite: inc p.idx
else:
p.idx = j # set back
result = "" # error
else:
result = ""
result = result.toLowerAscii()
result = ""
if currentTok(p).kind == tkWhite:
let (name, lastIdx) = getRefname(p, p.idx + 1)
let afterIdx = lastIdx + 1
if name.len > 0:
if p.tok[afterIdx].symbol == "::":
result = name
p.idx = afterIdx + 1
if currentTok(p).kind == tkWhite:
inc p.idx
elif currentTok(p).kind != tkIndent:
rstMessage(p, mwRstStyle,
"whitespace or newline expected after directive " & name)
result = result.toLowerAscii()
elif p.tok[afterIdx].symbol == ":":
rstMessage(p, mwRstStyle,
"double colon :: may be missing at end of '" & name & "'",
p.tok[afterIdx].line, p.tok[afterIdx].col)
elif p.tok[afterIdx].kind == tkPunct and p.tok[afterIdx].symbol[0] == ':':
rstMessage(p, mwRstStyle,
"too many colons for a directive (should be ::)",
p.tok[afterIdx].line, p.tok[afterIdx].col)
proc parseComment(p: var RstParser): PRstNode =
case currentTok(p).kind
@ -1711,7 +1752,8 @@ proc whichSection(p: RstParser): RstNodeKind =
return rnCodeBlock
elif currentTok(p).symbol == "::":
return rnLiteralBlock
elif currentTok(p).symbol == ".." and predNL(p):
elif currentTok(p).symbol == ".." and predNL(p) and
nextTok(p).kind in {tkWhite, tkIndent}:
return rnDirective
case currentTok(p).kind
of tkAdornment: