jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting; other bug fix (#14439)

Browse source 
* fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting
* remove orig deadcode from getAllRunnableExamplesImpl
* fix expected examples
* add test to close https://github.com/nim-lang/Nim/issues/14473
* correctly handle regular comments before 1st token inside runnableExamples
* add test to answer https://github.com/nim-lang/Nim/pull/14439#discussion_r431829199
* update tests
This commit is contained in:
Timothee Cour 2020-05-28 11:46:06 -07:00 • committed by GitHub
commit e013ebc91a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 927 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

1
compiler/ast.nim
View file

@ -1030,6 +1030,7 @@ const
nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
nkLambdaKinds* = {nkLambda, nkDo}
declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
procDefs* = nkLambdaKinds + declarativeDefs
nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}

159
compiler/docgen.nim
View file

@ -17,7 +17,7 @@ import
packages/docutils/rst, packages/docutils/rstgen,
json, xmltree, cgi, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, trees, tables, nimpaths
pathutils, trees, tables, nimpaths, renderverbatim
const
exportSection = skField
@ -315,7 +315,11 @@ proc genComment(d: PDoc, n: PNode): string =
result = ""
var dummyHasToc: bool
if n.comment.len > 0:
renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
var comment2 = n.comment
when false:
# RFC: to preseve newlines in comments, this would work:
comment2 = comment2.replace("\n", "\n\n")
renderRstToOut(d[], parseRst(comment2, toFullPath(d.conf, n.info),
toLinenumber(n.info), toColumn(n.info),
dummyHasToc, d.options, d.conf), result)
@ -494,8 +498,8 @@ proc runAllExamples(d: PDoc) =
rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
# removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
proc prepareExample(d: PDoc; n: PNode): string =
## returns `rdoccmd` for this runnableExamples
proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
## returns `rdoccmd` and source code for this runnableExamples
var rdoccmd = ""
if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
if n.len == 3:
@ -512,10 +516,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
docComment,
newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
runnableExamples.info = n.info
let ret = extractRunnableExamplesSource(d.conf, n)
for a in n.lastSon: runnableExamples.add a
# we could also use `ret` instead here, to keep sources verbatim
writeExample(d, runnableExamples, rdoccmd)
result = rdoccmd
result = (rdoccmd, ret)
when false:
proc extractImports(n: PNode; result: PNode) =
if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@ -529,39 +534,52 @@ proc prepareExample(d: PDoc; n: PNode): string =
for imp in imports: runnableExamples.add imp
runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) =
proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) =
## this is a rather hacky way to get rid of the initial indentation
## that the renderer currently produces:
# deadcode
var i = 0
var body = n.lastSon
if body.len == 1 and body.kind == nkStmtList and
body.lastSon.kind == nkStmtList:
body = body.lastSon
for b in body:
if i > 0: dest.add "\n"
inc i
nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
type RunnableState = enum
rsStart
rsComment
rsRunnable
rsDone
proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState =
##[
previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though).
This is to ensure this works:
Simple state machine to tell whether we render runnableExamples and doc comments.
This is to ensure that we can interleave runnableExamples and doc comments freely;
the logic is easy to change but currently a doc comment following another doc comment
will not render, to avoid rendering in following case:
proc fn* =
runnableExamples: discard
## d1
runnableExamples: discard
## d2
## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
It also works with:
proc fn* =
## d0
runnableExamples: discard
## d1
etc
## internal explanation # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
discard # some code
]##
# xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test
# now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample)
# if n.info.fileIndex != orig.info.fileIndex: return
case n.kind
of nkCommentStmt:
if previousIsRunnable:
if state in {rsStart, rsRunnable}:
dest.add genRecComment(d, n)
previousIsRunnable = false
return rsComment
of nkCallKinds:
if isRunnableExamples(n[0]) and
n.len >= 2 and n.lastSon.kind == nkStmtList:
previousIsRunnable = true
let rdoccmd = prepareExample(d, n)
n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
let (rdoccmd, code) = prepareExample(d, n)
var msg = "Example:"
if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
@ -569,27 +587,71 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous
inc d.listingCounter
let id = $d.listingCounter
dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
# this is a rather hacky way to get rid of the initial indentation
# that the renderer currently produces:
var i = 0
var body = n.lastSon
if body.len == 1 and body.kind == nkStmtList and
body.lastSon.kind == nkStmtList:
body = body.lastSon
for b in body:
if i > 0: dest.add "\n"
inc i
nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
when true:
var dest2 = ""
renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
dest.add dest2
else:
renderNimCodeOld(d, n, dest)
dest.add(d.config.getOrDefault"doc.listing_end" % id)
else: previousIsRunnable = false
return rsRunnable
else: discard
return rsDone
# change this to `rsStart` if you want to keep generating doc comments
# and runnableExamples that occur after some code in routine
var previousIsRunnable2 = true
for i in 0..<n.safeLen:
getAllRunnableExamplesRec(d, n[i], orig, dest, previousIsRunnable2)
proc getRoutineBody(n: PNode): PNode =
##[
nim transforms these quite differently:
proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
var previousIsRunnable = true
getAllRunnableExamplesRec(d, n, n, dest, previousIsRunnable)
proc someType*(): int =
## foo
result = 3
=>
result =
## foo
3;
proc someType*(): int =
## foo
3
=>
## foo
result = 3;
so we normalize the results to get to the statement list containing the
(0 or more) doc comments and runnableExamples.
(even if using `result = n[bodyPos]`, you'd still to apply similar logic).
]##
result = n[^1]
case result.kind
of nkSym:
result = n[^2]
case result.kind
of nkAsgn:
doAssert result[0].kind == nkSym
doAssert result.len == 2
result = result[1]
else: # eg: nkStmtList
discard
else:
discard
proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
var n = n
var state = rsStart
template fn(n2) =
state = getAllRunnableExamplesImpl(d, n2, dest, state)
case n.kind
of routineDefs:
n = n.getRoutineBody
case n.kind
of nkCommentStmt, nkCallKinds: fn(n)
else:
for i in 0..<n.safeLen:
fn(n[i])
if state == rsDone: return
else: fn(n)
proc isVisible(d: PDoc; n: PNode): bool =
result = false
@ -765,12 +827,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
var literal, plainName = ""
var kind = tkEof
var comm: Rope = nil
# skipping this (and doing it inside getAllRunnableExamples) would fix order in
# case of a runnableExample appearing before a doccomment, but would cause other
# issues
comm.add genRecComment(d, n)
if n.kind in declarativeDefs:
if n.kind in routineDefs:
getAllRunnableExamples(d, n, comm)
else:
comm.add genRecComment(d, n)
var r: TSrcGen
# Obtain the plain rendered string for hyperlink titles.
initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,

23
compiler/msgs.nim
View file

@ -436,18 +436,25 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
conf.m.fileInfos[fileIdx.int32].lines.add line
proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
if i.fileIndex.int32 < 0: return ""
if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
## it will return same number of lines (ie, a trailing empty line is discounted)
result = conf.m.fileInfos[fileIdx.int32].lines.len
if result == 0:
try:
for line in lines(toFullPathConsiderDirty(conf, i)):
addSourceLine conf, i.fileIndex, line.string
for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
addSourceLine conf, fileIdx, line.string
except IOError:
discard
assert i.fileIndex.int32 < conf.m.fileInfos.len
result = conf.m.fileInfos[fileIdx.int32].lines.len
proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
## 1-based index (matches editor line numbers); 1st line is for i.line = 1
## last valid line is `numLines` inclusive
if i.fileIndex.int32 < 0: return ""
let num = numLines(conf, i.fileIndex)
# can happen if the error points to EOF:
if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
if i.line.int > num: return ""
result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]

79
compiler/renderverbatim.nim Normal file
View file

@ -0,0 +1,79 @@
import strutils
from xmltree import addEscaped
import ast, options, msgs
import packages/docutils/highlite
# import compiler/renderer
import renderer
proc lastNodeRec(n: PNode): PNode =
result = n
while result.safeLen > 0: result = result[^1]
proc isInIndentationBlock(src: string, indent: int): bool =
#[
we stop at the first de-indentation; there's an inherent ambiguity with non
doc comments since they can have arbitrary indentation, so we just take the
practical route and require a runnableExamples to keep its code (including non
doc comments) to its indentation level.
]#
for j in 0..<indent:
if src.len <= j: return true
if src[j] != ' ': return false
return true
proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
## we'd need to check performance impact to enable it for nimdoc.
var first = n.lastSon.info
if first.line == n[0].info.line:
#[
runnableExamples: assert true
]#
discard
else:
#[
runnableExamples:
# non-doc comment that we want to capture even though `first` points to `assert true`
assert true
]#
first.line = n[0].info.line + 1
# first.col = n[0].info.col + 1 # anything with `col > n[0].col` is part of runnableExamples
let last = n.lastNodeRec.info
var info = first
var indent = info.col
let numLines = numLines(conf, info.fileIndex).uint16
var lastNonemptyPos = 0
for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample`
info.line = line
let src = sourceLine(conf, info)
if line > last.line and not isInIndentationBlock(src, indent):
break
if line > first.line: result.add "\n"
if src.len > indent:
result.add src[indent..^1]
lastNonemptyPos = result.len
result.setLen lastNonemptyPos
proc renderNimCode*(result: var string, code: string, isLatex = false) =
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
var buf = ""
template append(kind, val) =
buf.setLen 0
buf.addEscaped(val)
let class = tokenClassToStr[kind]
if isLatex:
result.addf "\\span$1{$2}" % [class, buf]
else:
result.addf "<span class=\"$1\">$2</span>" % [class, buf]
while true:
getNextToken(toknizr, langNim)
case toknizr.kind
of gtEof: break # End Of File (or string)
else:
# TODO: avoid alloc; maybe toOpenArray
append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))