jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

enable syntax highlighting for inline code (#17585)

Browse source 
* enable syntax highlighting for inline code

* finish '.. default-role' and preliminary '.. role'

implementation

* more compact check in dirRole

* set :literal: as default role for *.rst

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* use whichRole for setting currRoleKind

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

* rename rnGeneralRole -> rnUnknownRole

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>
This commit is contained in:
Andrey Makarov 2021-04-02 23:11:44 +03:00 • committed by GitHub
commit e35946f306
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 239 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

116
lib/packages/docutils/rst.nim
View file

@ -78,6 +78,13 @@
##
## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
## ``index`` [cmp:Sphinx]_
## * predefined roles ``:nim:`` (default), ``:c:`` (C programming language),
## ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
## That is every language that `highlite <highlite.html>`_ supports.
## They turn on appropriate syntax highlighting in inline code.
##
## .. Note:: default role for Nim files is ``:nim:``,
## for ``*.rst`` it's currently ``:literal:``.
##
## * ***triple emphasis*** (bold and italic) using \*\*\*
## * ``:idx:`` role for \`interpreted text\` to include the link to this
@ -161,7 +168,9 @@ type
roSupportSmilies, ## make the RST parser support smilies like ``:)``
roSupportRawDirective, ## support the ``raw`` directive (don't support
## it for sandboxing)
roSupportMarkdown ## support additional features of Markdown
roSupportMarkdown, ## support additional features of Markdown
roNimFile ## set for Nim files where default interpreted
## text role should be :nim:
RstParseOptions* = set[RstParseOption]
@ -454,6 +463,8 @@ type
hTitleCnt: int # =0 if no title, =1 if only main title,
# =2 if both title and subtitle are present
hCurLevel: int # current section level
currRole: string # current interpreted text role
currRoleKind: RstNodeKind # ... and its node kind
subs: seq[Substitution] # substitutions
refs: seq[Substitution] # references
anchors: seq[AnchorSubst] # internal target substitutions
@ -514,10 +525,36 @@ proc defaultFindFile*(filename: string): string =
if fileExists(filename): result = filename
else: result = ""
proc defaultRole(options: RstParseOptions): string =
if roNimFile in options: "nim" else: "literal"
# mirror highlite.nim sourceLanguageToStr with substitutions c++ cpp, c# csharp
const supportedLanguages = ["nim", "yaml", "python", "java", "c",
"cpp", "csharp"]
proc whichRoleAux(sym: string): RstNodeKind =
let r = sym.toLowerAscii
case r
of "idx": result = rnIdx
of "literal": result = rnInlineLiteral
of "strong": result = rnStrongEmphasis
of "emphasis": result = rnEmphasis
of "sub", "subscript": result = rnSub
of "sup", "superscript": result = rnSup
# literal and code are the same in our implementation
of "code": result = rnInlineLiteral
# c++ currently can be spelled only as cpp, c# only as csharp
elif r in supportedLanguages:
result = rnInlineCode
else: # unknown role
result = rnUnknownRole
proc newSharedState(options: RstParseOptions,
findFile: FindFileHandler,
msgHandler: MsgHandler): PSharedState =
new(result)
result.currRole = defaultRole(options)
result.currRoleKind = whichRoleAux(result.currRole)
result.subs = @[]
result.refs = @[]
result.options = options
@ -1018,15 +1055,28 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) =
for i in countup(0, sep - incr): a.add(n.sons[i])
for i in countup(sep + 1, n.len - 2): b.add(n.sons[i])
proc whichRole(sym: string): RstNodeKind =
case sym
of "idx": result = rnIdx
of "literal": result = rnInlineLiteral
of "strong": result = rnStrongEmphasis
of "emphasis": result = rnEmphasis
of "sub", "subscript": result = rnSub
of "sup", "superscript": result = rnSup
else: result = rnGeneralRole
proc whichRole(p: RstParser, sym: string): RstNodeKind =
result = whichRoleAux(sym)
if result == rnUnknownRole:
rstMessage(p, mwUnsupportedLanguage, p.s.currRole)
proc toInlineCode(n: PRstNode, language: string): PRstNode =
## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
result = newRstNode(rnInlineCode)
let args = newRstNode(rnDirArg)
var lang = language
if language == "cpp": lang = "c++"
elif language == "csharp": lang = "c#"
args.add newLeaf(lang)
result.add args
result.add PRstNode(nil)
var lb = newRstNode(rnLiteralBlock)
var s: string
for i in n.sons:
assert i.kind == rnLeaf
s.add i.text
lb.add newLeaf(s)
result.add lb
proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
var newKind = n.kind
@ -1052,14 +1102,23 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
result = newRstNode(newKind, newSons)
elif match(p, p.idx, ":w:"):
# a role:
newKind = whichRole(nextTok(p).symbol)
if newKind == rnGeneralRole:
let roleName = nextTok(p).symbol
newKind = whichRole(p, roleName)
if newKind == rnUnknownRole:
let newN = newRstNode(rnInner, n.sons)
newSons = @[newN, newLeaf(nextTok(p).symbol)]
newSons = @[newN, newLeaf(roleName)]
result = newRstNode(newKind, newSons)
elif newKind == rnInlineCode:
result = n.toInlineCode(language=roleName)
else:
result = newRstNode(newKind, newSons)
inc p.idx, 3
result = newRstNode(newKind, newSons)
else: # no change
result = n
else:
if p.s.currRoleKind == rnInlineCode:
result = n.toInlineCode(language=p.s.currRole)
else:
newKind = p.s.currRoleKind
result = newRstNode(newKind, newSons)
proc matchVerbatim(p: RstParser, start: int, expr: string): int =
result = start
@ -1315,9 +1374,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
parseUntil(p, n, "``", false)
father.add(n)
elif match(p, p.idx, ":w:") and p.tok[p.idx+3].symbol == "`":
let k = whichRole(nextTok(p).symbol)
let n = newRstNode(k)
let roleName = nextTok(p).symbol
let k = whichRole(p, roleName)
var n = newRstNode(k)
inc p.idx, 3
if k == rnInlineCode:
n = n.toInlineCode(language=roleName)
parseUntil(p, n, "`", false) # bug #17260
father.add(n)
elif isInlineMarkupStart(p, "`"):
@ -2421,6 +2483,18 @@ proc dirAdmonition(p: var RstParser, d: string): PRstNode =
proc dirDefaultRole(p: var RstParser): PRstNode =
result = parseDirective(p, rnDefaultRole, {hasArg}, nil)
if result.sons[0].len == 0: p.s.currRole = defaultRole(p.s.options)
else:
assert result.sons[0].sons[0].kind == rnLeaf
p.s.currRole = result.sons[0].sons[0].text
p.s.currRoleKind = whichRole(p, p.s.currRole)
proc dirRole(p: var RstParser): PRstNode =
result = parseDirective(p, rnDirective, {hasArg, hasOptions}, nil)
# just check that language is supported, TODO: real role association
let lang = getFieldValue(result, "language").strip
if lang != "" and lang notin supportedLanguages:
rstMessage(p, mwUnsupportedLanguage, lang)
proc dirRawAux(p: var RstParser, result: var PRstNode, kind: RstNodeKind,
contentParser: SectionParser) =
@ -2465,7 +2539,9 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
of "code-block": result = dirCodeBlock(p, nimExtension = true)
of "container": result = dirContainer(p)
of "contents": result = dirContents(p)
of "danger", "error": result = dirAdmonition(p, d)
of "danger": result = dirAdmonition(p, d)
of "default-role": result = dirDefaultRole(p)
of "error": result = dirAdmonition(p, d)
of "figure": result = dirFigure(p)
of "hint": result = dirAdmonition(p, d)
of "image": result = dirImage(p)
@ -2478,10 +2554,10 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
result = dirRaw(p)
else:
rstMessage(p, meInvalidDirective, d)
of "role": result = dirRole(p)
of "tip": result = dirAdmonition(p, d)
of "title": result = dirTitle(p)
of "warning": result = dirAdmonition(p, d)
of "default-role": result = dirDefaultRole(p)
else:
let tok = p.tok[p.idx-2] # report on directive in ".. directive::"
rstMessage(p, meInvalidDirective, d, tok.line, tok.col)

9
lib/packages/docutils/rstast.nim
View file

@ -55,12 +55,15 @@ type
# * `file#id <file#id>`_
# * `file#id <file#id>'_
rnSubstitutionDef, # a definition of a substitution
rnGeneralRole, # Inline markup:
# Inline markup:
rnInlineCode,
rnUnknownRole, # interpreted text with an unknown role
rnSub, rnSup, rnIdx,
rnEmphasis, # "*"
rnStrongEmphasis, # "**"
rnTripleEmphasis, # "***"
rnInterpretedText, # "`"
rnInterpretedText, # "`" an auxiliary role for parsing that will
# be converted into other kinds like rnInlineCode
rnInlineLiteral, # "``"
rnInlineTarget, # "_`target`"
rnSubstitutionReferences, # "|"
@ -252,7 +255,7 @@ proc renderRstToRst(d: var RenderContext, n: PRstNode, result: var string) =
result.add(" <")
renderRstToRst(d, n.sons[1], result)
result.add(">`_")
of rnGeneralRole:
of rnUnknownRole:
result.add('`')
renderRstToRst(d, n.sons[0],result)
result.add("`:")

33
lib/packages/docutils/rstgen.nim
View file

@ -942,7 +942,7 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
result.init
if n.isNil:
return
assert n.kind == rnCodeBlock
assert n.kind in {rnCodeBlock, rnInlineCode}
assert(not n.sons[2].isNil)
# Parse the field list for rendering parameters if there are any.
@ -987,8 +987,8 @@ proc buildLinesHtmlTable(d: PDoc; params: CodeBlockParams, code: string,
"</td></tr></tbody></table>" & (
d.config.getOrDefault"doc.listing_button" % id)
proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
## Renders a code block, appending it to `result`.
proc renderCode(d: PDoc, n: PRstNode, result: var string) =
## Renders a code (code block or inline code), appending it to `result`.
##
## If the code block uses the ``number-lines`` option, a table will be
## generated with two columns, the first being a list of numbers and the
@ -997,7 +997,7 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
## may also come from the parser through the internal ``default-language``
## option to differentiate between a plain code block and Nim's code block
## extension.
assert n.kind == rnCodeBlock
assert n.kind in {rnCodeBlock, rnInlineCode}
if n.sons[2] == nil: return
var params = d.parseCodeBlockParams(n)
var m = n.sons[2].sons[0]
@ -1006,10 +1006,23 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
if params.testCmd.len > 0 and d.onTestSnippet != nil:
d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text)
let (blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text,
var blockStart, blockEnd: string
case d.target
of outHtml:
if n.kind == rnCodeBlock:
(blockStart, blockEnd) = buildLinesHtmlTable(d, params, m.text,
n.anchor.idS)
dispA(d.target, result, blockStart,
"\\begin{rstpre}\n" & n.anchor.idS & "\n", [])
else: # rnInlineCode
blockStart = "<tt class=\"docutils literal\"><span class=\"pre\">"
blockEnd = "</span></tt>"
of outLatex:
if n.kind == rnCodeBlock:
blockStart = "\n\n\\begin{rstpre}" & n.anchor.idS & "\n"
blockEnd = "\n\\end{rstpre}\n"
else: # rnInlineCode
blockStart = "\\texttt{"
blockEnd = "}"
dispA(d.target, result, blockStart, blockStart, [])
if params.lang == langNone:
if len(params.langStr) > 0:
d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
@ -1028,7 +1041,7 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
tokenClassToStr[g.kind]])
deinitGeneralTokenizer(g)
dispA(d.target, result, blockEnd, "\n\\end{rstpre}\n")
dispA(d.target, result, blockEnd, blockEnd)
proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
var tmp = ""
@ -1294,13 +1307,13 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
result.add addNodes(lastSon(n))
of rnImage, rnFigure: renderImage(d, n, result)
of rnCodeBlock: renderCodeBlock(d, n, result)
of rnCodeBlock, rnInlineCode: renderCode(d, n, result)
of rnContainer: renderContainer(d, n, result)
of rnSubstitutionReferences, rnSubstitutionDef:
renderAux(d, n, "|$1|", "|$1|", result)
of rnDirective:
renderAux(d, n, "", "", result)
of rnGeneralRole:
of rnUnknownRole:
var tmp0 = ""
var tmp1 = ""
renderRstToOut(d, n.sons[0], tmp0)