the documentation generator now supports ':test:' for the testing of test snippets

changelog.md

@@ -98,3 +98,10 @@ This now needs to be written as:
 - Added ``system.runnableExamples`` to make examples in Nim's documentation easier
   to write and test. The examples are tested as the last step of
   ``nim doc``.
+- Nim's ``rst2html`` command now supports the testing of code snippets via an RST
+  extension that we called ``:test:``::
+
+    .. code-block:: nim
+        :test:
+      # shows how the 'if' statement works
+      if true: echo "yes"

compiler/docgen.nim

@@ -22,7 +22,6 @@ type
   TSections = array[TSymKind, Rope]
   TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope           # module description
-    id: int                  # for generating IDs
     toc, section: TSections
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
@@ -109,6 +108,8 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc =
   result.id = 100
   result.jArray = newJArray()
   initStrTable result.types
+  result.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) =
+    localError(newLineInfo(d.filename, -1, -1), warnUser, "only 'rst2html' supports the ':test:' attribute")
 
 proc dispA(dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if gCmd != cmdRst2tex: addf(dest, xml, args)
@@ -274,7 +275,9 @@ proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
       # that the renderer currently produces:
       var i = 0
       var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList: body = body.lastSon
+      if body.len == 1 and body.kind == nkStmtList and
+          body.lastSon.kind == nkStmtList:
+        body = body.lastSon
       for b in body:
         if i > 0: dest.add "\n"
         inc i
@@ -785,6 +788,23 @@ proc commandDoc*() =
 proc commandRstAux(filename, outExt: string) =
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, options.gConfigVars)
+  d.onTestSnippet = proc (d: var RstGenerator; filename, cmd: string;
+                          status: int; content: string) =
+    var outp: string
+    if filename.len == 0:
+      inc(d.id)
+      outp = getNimcacheDir() / splitFile(d.filename).name & "_snippet_" & $d.id & ".nim"
+    elif isAbsolute(filename):
+      outp = filename
+    else:
+      # Nim's convention: every path is relative to the file it was written in:
+      outp = splitFile(d.filename).dir / filename
+    writeFile(outp, content)
+    let cmd = cmd % outp
+    rawMessage(hintExecuting, cmd)
+    if execShellCmd(cmd) != status:
+      rawMessage(errExecutionOfProgramFailed, cmd)
+
   d.isPureRst = true
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})

compiler/semexprs.nim

@@ -1849,13 +1849,14 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
         result = magicsAfterOverloadResolution(c, result, flags)
   of mRunnableExamples:
     if gCmd == cmdDoc and n.len >= 2 and n.lastSon.kind == nkStmtList:
-      if sfMainModule in c.module.flags:
+      if n.sons[0].kind == nkIdent:
-        let inp = toFullPath(c.module.info)
+        if sfMainModule in c.module.flags:
-        if c.runnableExamples == nil:
+          let inp = toFullPath(c.module.info)
-          c.runnableExamples = newTree(nkStmtList,
+          if c.runnableExamples == nil:
-            newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
+            c.runnableExamples = newTree(nkStmtList,
-        c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+              newTree(nkImportStmt, newStrNode(nkStrLit, expandFilename(inp))))
-      result = setMs(n, s)
+          c.runnableExamples.add newTree(nkBlockStmt, emptyNode, copyTree n.lastSon)
+        result = setMs(n, s)
     else:
       result = emptyNode
   else:

compiler/transf.nim

@@ -693,7 +693,7 @@ proc transformCall(c: PTransf, n: PNode): PTransNode =
           inc(j)
       add(result, a.PTransNode)
     if len(result) == 2: result = result[1]
-  elif magic in {mNBindSym, mTypeOf}:
+  elif magic in {mNBindSym, mTypeOf, mRunnableExamples}:
     # for bindSym(myconst) we MUST NOT perform constant folding:
     result = n.PTransNode
   elif magic == mProcCall:

lib/packages/docutils/highlite.nim

@@ -31,14 +31,12 @@ type
     state: TokenClass
 
   SourceLanguage* = enum
-    langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava,
+    langNone, langNim, langCpp, langCsharp, langC, langJava,
     langYaml
-{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass,
-              TGeneralTokenizer: GeneralTokenizer].}
 
 const
   sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
-    "Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"]
+    "Nim", "C++", "C#", "C", "Java", "Yaml"]
   tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
     "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
     "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
@@ -398,7 +396,6 @@ type
   TokenizerFlag = enum
     hasPreprocessor, hasNestedComments
   TokenizerFlags = set[TokenizerFlag]
-{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].}
 
 proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
                     flags: TokenizerFlags) =
@@ -888,7 +885,7 @@ proc yamlNextToken(g: var GeneralTokenizer) =
 proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   case lang
   of langNone: assert false
-  of langNim, langNimrod: nimNextToken(g)
+  of langNim: nimNextToken(g)
   of langCpp: cppNextToken(g)
   of langCsharp: csharpNextToken(g)
   of langC: cNextToken(g)

lib/packages/docutils/rst.nim

@@ -45,8 +45,6 @@ type
   MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind,
                        arg: string) {.nimcall.} ## what to do in case of an error
   FindFileHandler* = proc (filename: string): string {.nimcall.}
-{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption,
-              TMsgKind: MsgKind].}
 
 const
   messages: array[MsgKind, string] = [
@@ -127,8 +125,6 @@ type
     bufpos*: int
     line*, col*, baseIndent*: int
     skipPounds*: bool
-{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq,
-              TLexer: Lexer].}
 
 proc getThing(L: var Lexer, tok: var Token, s: set[char]) =
   tok.kind = tkWord
@@ -288,10 +284,6 @@ type
     hasToc*: bool
 
   EParseError* = object of ValueError
-{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution,
-              TSharedState: SharedState, TRstParser: RstParser,
-              TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler,
-              TMsgClass: MsgClass].}
 
 proc whichMsgClass*(k: MsgKind): MsgClass =
   ## returns which message class `k` belongs to.
@@ -341,11 +333,6 @@ proc rstMessage(p: RstParser, msgKind: MsgKind) =
                              p.col + p.tok[p.idx].col, msgKind,
                              p.tok[p.idx].symbol)
 
-when false:
-  proc corrupt(p: RstParser) =
-    assert p.indentStack[0] == 0
-    for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000
-
 proc currInd(p: RstParser): int =
   result = p.indentStack[high(p.indentStack)]
 

lib/packages/docutils/rstgen.nim

@@ -61,6 +61,9 @@ type
     seenIndexTerms: Table[string, int] ## \
     ## Keeps count of same text index terms to generate different identifiers
     ## for hyperlinks. See renderIndexTerm proc for details.
+    id*: int               ## A counter useful for generating IDs.
+    onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
+                          content: string)
 
   PDoc = var RstGenerator ## Alias to type less.
 
@@ -69,8 +72,9 @@ type
     startLine: int ## The starting line of the code block, by default 1.
     langStr: string ## Input string used to specify the language.
     lang: SourceLanguage ## Type of highlighting, by default none.
-{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry,
+    filename: string
-              TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].}
+    testCmd: string
+    status: int
 
 proc init(p: var CodeBlockParams) =
   ## Default initialisation of CodeBlockParams to sane values.
@@ -133,6 +137,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.options = options
   g.findFile = findFile
   g.currentSection = ""
+  g.id = 0
   let fileParts = filename.splitFile
   if fileParts.ext == ".nim":
     g.currentSection = "Module " & fileParts.name
@@ -368,7 +373,6 @@ type
     ##
     ## The value indexed by this IndexEntry is a sequence with the real index
     ## entries found in the ``.idx`` file.
-{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].}
 
 proc cmp(a, b: IndexEntry): int =
   ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`.
@@ -823,13 +827,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
     var number: int
     if parseInt(n.getFieldValue, number) > 0:
       params.startLine = number
-  of "file":
+  of "file", "filename":
     # The ``file`` option is a Nim extension to the official spec, it acts
     # like it would for other directives like ``raw`` or ``cvs-table``. This
     # field is dealt with in ``rst.nim`` which replaces the existing block with
     # the referenced file, so we only need to ignore it here to avoid incorrect
     # warning messages.
-    discard
+    params.filename = n.getFieldValue.strip
+  of "test":
+    params.testCmd = n.getFieldValue.strip
+    if params.testCmd.len == 0: params.testCmd = "nim c -r $1"
+  of "status":
+    var status: int
+    if parseInt(n.getFieldValue, status) > 0:
+      params.status = status
   of "default-language":
     params.langStr = n.getFieldValue.strip
     params.lang = params.langStr.getSourceLanguage
@@ -901,6 +912,9 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
   var m = n.sons[2].sons[0]
   assert m.kind == rnLeaf
 
+  if params.testCmd.len > 0 and d.onTestSnippet != nil:
+    d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text)
+
   let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text)
 
   dispA(d.target, result, blockStart, "\\begin{rstpre}\n", [])