extracted documentation generator

compiler/ccgmerge.nim

@@ -169,7 +169,7 @@ proc readVerbatimSection(L: var TBaseLexer): PRope =
       result.data.add(buf[pos])
       inc pos
   L.bufpos = pos
-  result.length = result.data.len
+  freezeMutableRope(result)
 
 proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos

compiler/commands.nim

@@ -366,8 +366,7 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
     expectArg(switch, arg, pass, info)
     if pass in {passCmd2, passPP}: cLinkedLibs.add arg
   of "index": 
-    expectArg(switch, arg, pass, info)
+    ProcessOnOffSwitchG({optGenIndex}, arg, pass, info)
-    if pass in {passCmd2, passPP}: gIndexFile = arg
   of "import":
     expectArg(switch, arg, pass, info)
     if pass in {passCmd2, passPP}: implicitImports.add arg

compiler/docgen.nim

@@ -12,57 +12,20 @@
 # by knowing how the anchors are going to be named.
 
 import 
-  ast, astalgo, strutils, hashes, options, nversion, msgs, os, ropes, idents, 
+  ast, strutils, strtabs, options, msgs, os, ropes, idents, 
   wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite, 
   importer
 
-proc CommandDoc*()
-proc CommandRst2Html*()
-proc CommandRst2TeX*()
-
-#proc CommandBuildIndex*()
-# implementation
-
 type
-  TTocEntry{.final.} = object 
-    n*: PRstNode
-    refname*, header*: PRope
-
   TSections = array[TSymKind, PRope]
-  TMetaEnum = enum 
+  TDocumentor = object of rstgen.TRstGenerator
-    metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion
+    modDesc: PRope           # module description
-  TDocumentor {.final.} = object # contains a module's documentation
+    id: int                  # for generating IDs
-    target: TOutputTarget
+    toc, section: TSections
-    options: TRstParseOptions
+    indexValFilename: string
-    filename*: string         # filename of the source file; without extension
-    modDesc*: PRope           # module description
-    id*: int                  # for generating IDs
-    splitAfter*: int          # split too long entries in the TOC
-    tocPart*: seq[TTocEntry]
-    hasToc*: bool
-    toc*, section*: TSections
-    indexFile*, theIndex*: PRstNode
-    indexValFilename*: string
-    meta*: array[TMetaEnum, PRope]
 
   PDoc = ref TDocumentor
   
-proc findIndexNode(n: PRstNode): PRstNode = 
-  if n == nil: 
-    result = nil
-  elif n.kind == rnIndex: 
-    result = n.sons[2]
-    if result == nil: 
-      result = newRstNode(rnDefList)
-      n.sons[2] = result
-    elif result.kind == rnInner: 
-      result = result.sons[0]
-  else: 
-    result = nil
-    for i in countup(0, len(n) - 1): 
-      result = findIndexNode(n.sons[i])
-      if result != nil: return 
-  
 proc compilerMsgHandler(filename: string, line, col: int,
                         msgKind: rst.TMsgKind, arg: string) {.procvar.} =
   # translate msg kind:
@@ -76,6 +39,7 @@ proc compilerMsgHandler(filename: string, line, col: int,
   of meInvalidDirective: k = errInvalidDirectiveX
   of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel
   of mwUnknownSubstitution: k = warnUnknownSubstitutionX
+  of mwUnsupportedLanguage: k = warnLanguageXNotSupported
   GlobalError(newLineInfo(filename, line, col), k, arg)
   
 proc parseRst(text, filename: string,
@@ -84,44 +48,16 @@ proc parseRst(text, filename: string,
   result = rstParse(text, filename, line, column, hasToc, rstOptions,
                     options.FindFile, compilerMsgHandler)
 
-proc initIndexFile(d: PDoc) = 
+proc newDocumentor(filename: string, config: PStringTable): PDoc = 
-  var 
-    h: PRstNode
-    dummyHasToc: bool
-  if gIndexFile.len == 0: return 
-  gIndexFile = addFileExt(gIndexFile, "txt")
-  d.indexValFilename = changeFileExt(extractFilename(d.filename), HtmlExt)
-  if ExistsFile(gIndexFile): 
-    d.indexFile = parseRst(readFile(gIndexFile), gIndexFile, 0, 1, 
-                           dummyHasToc, {roSupportRawDirective})
-    d.theIndex = findIndexNode(d.indexFile)
-    if (d.theIndex == nil) or (d.theIndex.kind != rnDefList): 
-      rawMessage(errXisNoValidIndexFile, gIndexFile)
-    clearIndex(d.theIndex, d.indexValFilename)
-  else: 
-    d.indexFile = newRstNode(rnInner)
-    h = newRstNode(rnOverline)
-    h.level = 1
-    add(h, newRstNode(rnLeaf, "Index"))
-    add(d.indexFile, h)
-    h = newRstNode(rnIndex)
-    add(h, nil)            # no argument
-    add(h, nil)            # no options
-    d.theIndex = newRstNode(rnDefList)
-    add(h, d.theIndex)
-    add(d.indexFile, h)
-
-proc newDocumentor(filename: string): PDoc = 
   new(result)
-  if gCmd != cmdRst2Tex: result.target = outHtml
+  initRstGenerator(result[], (if gCmd != cmdRst2Tex: outHtml else: outLatex),
-  else: result.target = outLatex
+                   options.gConfigVars, filename, {roSupportRawDirective},
-  result.tocPart = @[]
+                   options.FindFile, compilerMsgHandler)
-  result.filename = filename
   result.id = 100
-  result.splitAfter = 20
+
-  result.options = {roSupportRawDirective}
+proc dispA(dest: var PRope, xml, tex: string, args: openarray[PRope]) = 
-  var s = getConfigVar("split.item.toc")
+  if gCmd != cmdRst2Tex: appf(dest, xml, args)
-  if s != "": result.splitAfter = parseInt(s)
+  else: appf(dest, tex, args)
   
 proc getVarIdx(varnames: openarray[string], id: string): int = 
   for i in countup(0, high(varnames)): 
@@ -183,59 +119,20 @@ proc ropeFormatNamedVars(frmt: TFormatStr, varnames: openarray[string],
       else: break 
     if i - 1 >= start: app(result, substr(frmt, start, i - 1))
 
-proc disp(xml, tex: string): string = 
+proc genComment(d: PDoc, n: PNode): string =
-  if gCmd != cmdRst2Tex: result = xml
+  result = ""
-  else: result = tex
-  
-proc dispF(xml, tex: string, args: openarray[PRope]): PRope = 
-  if gCmd != cmdRst2Tex: result = ropef(xml, args)
-  else: result = ropef(tex, args)
-  
-proc dispA(dest: var PRope, xml, tex: string, args: openarray[PRope]) = 
-  if gCmd != cmdRst2Tex: appf(dest, xml, args)
-  else: appf(dest, tex, args)
-  
-proc renderRstToOut(d: PDoc, n: PRstNode): PRope
-
-proc renderAux(d: PDoc, n: PRstNode, outer: string = "$1"): PRope = 
-  result = nil
-  for i in countup(0, len(n) - 1): app(result, renderRstToOut(d, n.sons[i]))
-  result = ropef(outer, [result])
-
-proc setIndexForSourceTerm(d: PDoc, name: PRstNode, id: int) = 
-  if d.theIndex == nil: return 
-  var h = newRstNode(rnHyperlink)
-  var a = newRstNode(rnLeaf, d.indexValFilename & disp("#", "") & $id)
-  add(h, a)
-  add(h, a)
-  a = newRstNode(rnIdx)
-  add(a, name)
-  setIndexPair(d.theIndex, a, h)
-
-proc renderIndexTerm(d: PDoc, n: PRstNode): PRope = 
-  inc(d.id)
-  result = dispF("<span id=\"$1\">$2</span>", "$2\\label{$1}", 
-                 [toRope(d.id), renderAux(d, n)])
-  var h = newRstNode(rnHyperlink)
-  var a = newRstNode(rnLeaf, d.indexValFilename & disp("#", "") & $d.id)
-  add(h, a)
-  add(h, a)
-  setIndexPair(d.theIndex, n, h)
-
-proc genComment(d: PDoc, n: PNode): PRope = 
   var dummyHasToc: bool
   if n.comment != nil and startsWith(n.comment, "##"):
-    result = renderRstToOut(d, parseRst(n.comment, toFilename(n.info),
+    renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
                                toLineNumber(n.info), toColumn(n.info), 
-                                        dummyHasToc, 
+                               dummyHasToc, d.options + {roSkipPounds}), result)
-                                        d.options + {roSkipPounds}))
 
 proc genRecComment(d: PDoc, n: PNode): PRope = 
   if n == nil: return nil
-  result = genComment(d, n)
+  result = genComment(d, n).toRope
   if result == nil: 
-    if not (n.kind in {nkEmpty..nkNilLit}): 
+    if n.kind notin {nkEmpty..nkNilLit}:
-      for i in countup(0, sonsLen(n) - 1): 
+      for i in countup(0, len(n)-1):
         result = genRecComment(d, n.sons[i])
         if result != nil: return 
   else:
@@ -244,15 +141,15 @@ proc genRecComment(d: PDoc, n: PNode): PRope =
 proc isVisible(n: PNode): bool = 
   result = false
   if n.kind == nkPostfix: 
-    if (sonsLen(n) == 2) and (n.sons[0].kind == nkIdent): 
+    if n.len == 2 and n.sons[0].kind == nkIdent: 
       var v = n.sons[0].ident
-      result = (v.id == ord(wStar)) or (v.id == ord(wMinus))
+      result = v.id == ord(wStar) or v.id == ord(wMinus)
   elif n.kind == nkSym:
     result = sfExported in n.sym.flags
   elif n.kind == nkPragmaExpr:
     result = isVisible(n.sons[0])
   
-proc getName(d: PDoc, n: PNode, splitAfter: int = - 1): string = 
+proc getName(d: PDoc, n: PNode, splitAfter = -1): string = 
   case n.kind
   of nkPostfix: result = getName(d, n.sons[1], splitAfter)
   of nkPragmaExpr: result = getName(d, n.sons[0], splitAfter)
@@ -334,287 +231,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
   app(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"), 
                                     ["name", "header", "desc", "itemID"], [
       toRope(getName(d, nameNode, d.splitAfter)), result, comm, toRope(d.id)]))
-  setIndexForSourceTerm(d, getRstName(nameNode), d.id)
+  setIndexTerm(d[], $d.id, getName(d, nameNode))
-
-proc renderHeadline(d: PDoc, n: PRstNode): PRope = 
-  result = nil
-  for i in countup(0, len(n) - 1): app(result, renderRstToOut(d, n.sons[i]))
-  var refname = toRope(rstnodeToRefname(n))
-  if d.hasToc: 
-    var length = len(d.tocPart)
-    setlen(d.tocPart, length + 1)
-    d.tocPart[length].refname = refname
-    d.tocPart[length].n = n
-    d.tocPart[length].header = result
-    result = dispF(
-        "<h$1><a class=\"toc-backref\" id=\"$2\" href=\"#$2_toc\">$3</a></h$1>", 
-        "\\rsth$4{$3}\\label{$2}$n", [toRope(n.level), 
-        d.tocPart[length].refname, result, 
-        toRope(chr(n.level - 1 + ord('A')) & "")])
-  else: 
-    result = dispF("<h$1 id=\"$2\">$3</h$1>", "\\rsth$4{$3}\\label{$2}$n", [
-        toRope(n.level), refname, result, 
-        toRope(chr(n.level - 1 + ord('A')) & "")])
-  
-proc renderOverline(d: PDoc, n: PRstNode): PRope = 
-  var t: PRope = nil
-  for i in countup(0, len(n) - 1): app(t, renderRstToOut(d, n.sons[i]))
-  result = nil
-  if d.meta[metaTitle] == nil: 
-    d.meta[metaTitle] = t
-  elif d.meta[metaSubtitle] == nil: 
-    d.meta[metaSubtitle] = t
-  else: 
-    result = dispF("<h$1 id=\"$2\"><center>$3</center></h$1>", 
-                   "\\rstov$4{$3}\\label{$2}$n", [toRope(n.level), 
-        toRope(rstnodeToRefname(n)), t, toRope($chr(n.level - 1 + ord('A')))])
-  
-
-proc renderTocEntry(d: PDoc, e: TTocEntry): PRope = 
-  result = dispF(
-    "<li><a class=\"reference\" id=\"$1_toc\" href=\"#$1\">$2</a></li>$n", 
-    "\\item\\label{$1_toc} $2\\ref{$1}$n", [e.refname, e.header])
-
-proc renderTocEntries(d: PDoc, j: var int, lvl: int): PRope = 
-  result = nil
-  while j <= high(d.tocPart): 
-    var a = abs(d.tocPart[j].n.level)
-    if a == lvl: 
-      app(result, renderTocEntry(d, d.tocPart[j]))
-      inc(j)
-    elif a > lvl: 
-      app(result, renderTocEntries(d, j, a))
-    else: 
-      break 
-  if lvl > 1: 
-    result = dispF("<ul class=\"simple\">$1</ul>", 
-                   "\\begin{enumerate}$1\\end{enumerate}", [result])
-  
-proc fieldAux(s: string): PRope = 
-  result = toRope(strip(s))
-
-proc renderImage(d: PDoc, n: PRstNode): PRope = 
-  var options: PRope = nil
-  var s = getFieldValue(n, "scale")
-  if s != "": dispA(options, " scale=\"$1\"", " scale=$1", [fieldAux(s)])
-  s = getFieldValue(n, "height")
-  if s != "": dispA(options, " height=\"$1\"", " height=$1", [fieldAux(s)])
-  s = getFieldValue(n, "width")
-  if s != "": dispA(options, " width=\"$1\"", " width=$1", [fieldAux(s)])
-  s = getFieldValue(n, "alt")
-  if s != "": dispA(options, " alt=\"$1\"", "", [fieldAux(s)])
-  s = getFieldValue(n, "align")
-  if s != "": dispA(options, " align=\"$1\"", "", [fieldAux(s)])
-  if options != nil: options = dispF("$1", "[$1]", [options])
-  result = dispF("<img src=\"$1\"$2 />", "\\includegraphics$2{$1}", 
-                 [toRope(getArgument(n)), options])
-  if len(n) >= 3: app(result, renderRstToOut(d, n.sons[2]))
-  
-proc renderSmiley(d: PDoc, n: PRstNode): PRope =
-  result = dispF(
-    """<img src="images/smilies/$1.gif" width="15" 
-        height="17" hspace="2" vspace="2" />""",
-    "\\includegraphics{$1}", [toRope(n.text)])
-  
-proc renderCodeBlock(d: PDoc, n: PRstNode): PRope = 
-  result = nil
-  if n.sons[2] == nil: return 
-  var m = n.sons[2].sons[0]
-  if (m.kind != rnLeaf): InternalError("renderCodeBlock")
-  var langstr = strip(getArgument(n))
-  var lang: TSourceLanguage
-  if langstr == "": 
-    lang = langNimrod         # default language
-  else: 
-    lang = getSourceLanguage(langstr)
-  if lang == langNone: 
-    rawMessage(warnLanguageXNotSupported, langstr)
-    result = toRope(m.text)
-  else: 
-    var g: TGeneralTokenizer
-    initGeneralTokenizer(g, m.text)
-    while true: 
-      getNextToken(g, lang)
-      case g.kind
-      of gtEof: break 
-      of gtNone, gtWhitespace: 
-        app(result, substr(m.text, g.start, g.length + g.start - 1))
-      else: 
-        dispA(result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
-          toRope(esc(d.target, substr(m.text, g.start, g.length+g.start-1))),
-          toRope(tokenClassToStr[g.kind])])
-    deinitGeneralTokenizer(g)
-  if result != nil:
-    result = dispF("<pre>$1</pre>", "\\begin{rstpre}$n$1$n\\end{rstpre}$n", 
-                   [result])
-  
-proc renderContainer(d: PDoc, n: PRstNode): PRope = 
-  result = renderRstToOut(d, n.sons[2])
-  var arg = toRope(strip(getArgument(n)))
-  if arg == nil: result = dispF("<div>$1</div>", "$1", [result])
-  else: result = dispF("<div class=\"$1\">$2</div>", "$2", [arg, result])
-  
-proc texColumns(n: PRstNode): string = 
-  result = ""
-  for i in countup(1, len(n)): add(result, "|X")
-  
-proc renderField(d: PDoc, n: PRstNode): PRope = 
-  var b = false
-  if gCmd == cmdRst2Tex: 
-    var fieldname = addNodes(n.sons[0])
-    var fieldval = toRope(esc(d.target, strip(addNodes(n.sons[1]))))
-    if cmpIgnoreStyle(fieldname, "author") == 0: 
-      if d.meta[metaAuthor] == nil: 
-        d.meta[metaAuthor] = fieldval
-        b = true
-    elif cmpIgnoreStyle(fieldName, "version") == 0: 
-      if d.meta[metaVersion] == nil: 
-        d.meta[metaVersion] = fieldval
-        b = true
-  if b: result = nil
-  else: result = renderAux(d, n, disp("<tr>$1</tr>$n", "$1"))
-  
-proc renderRstToOut(d: PDoc, n: PRstNode): PRope = 
-  if n == nil: 
-    return nil
-  case n.kind
-  of rnInner: result = renderAux(d, n)
-  of rnHeadline: result = renderHeadline(d, n)
-  of rnOverline: result = renderOverline(d, n)
-  of rnTransition: result = renderAux(d, n, disp("<hr />\n", "\\hrule\n"))
-  of rnParagraph: result = renderAux(d, n, disp("<p>$1</p>\n", "$1$n$n"))
-  of rnBulletList: 
-    result = renderAux(d, n, disp("<ul class=\"simple\">$1</ul>\n", 
-                                  "\\begin{itemize}$1\\end{itemize}\n"))
-  of rnBulletItem, rnEnumItem: 
-    result = renderAux(d, n, disp("<li>$1</li>\n", "\\item $1\n"))
-  of rnEnumList: 
-    result = renderAux(d, n, disp("<ol class=\"simple\">$1</ol>\n", 
-                                  "\\begin{enumerate}$1\\end{enumerate}\n"))
-  of rnDefList: 
-    result = renderAux(d, n, disp("<dl class=\"docutils\">$1</dl>\n", 
-                       "\\begin{description}$1\\end{description}\n"))
-  of rnDefItem: result = renderAux(d, n)
-  of rnDefName: result = renderAux(d, n, disp("<dt>$1</dt>\n", "\\item[$1] "))
-  of rnDefBody: result = renderAux(d, n, disp("<dd>$1</dd>\n", "$1\n"))
-  of rnFieldList: 
-    result = nil
-    for i in countup(0, len(n) - 1): 
-      app(result, renderRstToOut(d, n.sons[i]))
-    if result != nil: 
-      result = dispf(
-          "<table class=\"docinfo\" frame=\"void\" rules=\"none\">" &
-          "<col class=\"docinfo-name\" />" &
-          "<col class=\"docinfo-content\" />" & 
-          "<tbody valign=\"top\">$1" &
-          "</tbody></table>", 
-          "\\begin{description}$1\\end{description}\n", 
-          [result])
-  of rnField: result = renderField(d, n)
-  of rnFieldName: 
-    result = renderAux(d, n, disp("<th class=\"docinfo-name\">$1:</th>", 
-                                  "\\item[$1:]"))
-  of rnFieldBody: 
-    result = renderAux(d, n, disp("<td>$1</td>", " $1$n"))
-  of rnIndex: 
-    result = renderRstToOut(d, n.sons[2])
-  of rnOptionList: 
-    result = renderAux(d, n, disp("<table frame=\"void\">$1</table>", 
-      "\\begin{description}$n$1\\end{description}\n"))
-  of rnOptionListItem: 
-    result = renderAux(d, n, disp("<tr>$1</tr>$n", "$1"))
-  of rnOptionGroup: 
-    result = renderAux(d, n, disp("<th align=\"left\">$1</th>", "\\item[$1]"))
-  of rnDescription: 
-    result = renderAux(d, n, disp("<td align=\"left\">$1</td>$n", " $1$n"))
-  of rnOption, rnOptionString, rnOptionArgument: 
-    InternalError("renderRstToOut")
-  of rnLiteralBlock: 
-    result = renderAux(d, n, disp("<pre>$1</pre>$n", 
-                                  "\\begin{rstpre}$n$1$n\\end{rstpre}$n"))
-  of rnQuotedLiteralBlock: 
-    InternalError("renderRstToOut")
-  of rnLineBlock: 
-    result = renderAux(d, n, disp("<p>$1</p>", "$1$n$n"))
-  of rnLineBlockItem: 
-    result = renderAux(d, n, disp("$1<br />", "$1\\\\$n"))
-  of rnBlockQuote: 
-    result = renderAux(d, n, disp("<blockquote><p>$1</p></blockquote>$n", 
-                                  "\\begin{quote}$1\\end{quote}$n"))
-  of rnTable, rnGridTable: 
-    result = renderAux(d, n, disp(
-      "<table border=\"1\" class=\"docutils\">$1</table>", 
-      "\\begin{table}\\begin{rsttab}{" &
-        texColumns(n) & "|}$n\\hline$n$1\\end{rsttab}\\end{table}"))
-  of rnTableRow: 
-    if len(n) >= 1: 
-      result = renderRstToOut(d, n.sons[0])
-      for i in countup(1, len(n) - 1): 
-        dispa(result, "$1", " & $1", [renderRstToOut(d, n.sons[i])])
-      result = dispf("<tr>$1</tr>$n", "$1\\\\$n\\hline$n", [result])
-    else: 
-      result = nil
-  of rnTableDataCell: result = renderAux(d, n, disp("<td>$1</td>", "$1"))
-  of rnTableHeaderCell: 
-    result = renderAux(d, n, disp("<th>$1</th>", "\\textbf{$1}"))
-  of rnLabel: 
-    InternalError("renderRstToOut") # used for footnotes and other
-  of rnFootnote: 
-    InternalError("renderRstToOut") # a footnote
-  of rnCitation: 
-    InternalError("renderRstToOut") # similar to footnote
-  of rnRef: 
-    result = dispF("<a class=\"reference external\" href=\"#$2\">$1</a>", 
-                   "$1\\ref{$2}", [renderAux(d, n), toRope(rstnodeToRefname(n))])
-  of rnStandaloneHyperlink: 
-    result = renderAux(d, n, disp(
-      "<a class=\"reference external\" href=\"$1\">$1</a>", 
-      "\\href{$1}{$1}"))
-  of rnHyperlink: 
-    result = dispF("<a class=\"reference external\" href=\"$2\">$1</a>", 
-                   "\\href{$2}{$1}", 
-                   [renderRstToOut(d, n.sons[0]), renderRstToOut(d, n.sons[1])])
-  of rnDirArg, rnRaw: result = renderAux(d, n)
-  of rnRawHtml: 
-    if gCmd != cmdRst2Tex:
-      result = toRope(addNodes(lastSon(n)))
-  of rnRawLatex:
-    if gCmd == cmdRst2Tex:
-      result = toRope(addNodes(lastSon(n)))
-      
-  of rnImage, rnFigure: result = renderImage(d, n)
-  of rnCodeBlock: result = renderCodeBlock(d, n)
-  of rnContainer: result = renderContainer(d, n)
-  of rnSubstitutionReferences, rnSubstitutionDef: 
-    result = renderAux(d, n, disp("|$1|", "|$1|"))
-  of rnDirective: 
-    result = renderAux(d, n, "") # Inline markup:
-  of rnGeneralRole: 
-    result = dispF("<span class=\"$2\">$1</span>", "\\span$2{$1}", 
-                   [renderRstToOut(d, n.sons[0]), renderRstToOut(d, n.sons[1])])
-  of rnSub: result = renderAux(d, n, disp("<sub>$1</sub>", "\\rstsub{$1}"))
-  of rnSup: result = renderAux(d, n, disp("<sup>$1</sup>", "\\rstsup{$1}"))
-  of rnEmphasis: result = renderAux(d, n, disp("<em>$1</em>", "\\emph{$1}"))
-  of rnStrongEmphasis:
-    result = renderAux(d, n, disp("<strong>$1</strong>", "\\textbf{$1}"))
-  of rnTripleEmphasis:
-    result = renderAux(d, n, disp("<strong><em>$1</em></strong>", 
-                                  "\\textbf{emph{$1}}"))
-  of rnInterpretedText:
-    result = renderAux(d, n, disp("<cite>$1</cite>", "\\emph{$1}"))
-  of rnIdx: 
-    if d.theIndex == nil: 
-      result = renderAux(d, n, disp("<span>$1</span>", "\\emph{$1}"))
-    else: 
-      result = renderIndexTerm(d, n)
-  of rnInlineLiteral: 
-    result = renderAux(d, n, disp(
-      "<tt class=\"docutils literal\"><span class=\"pre\">$1</span></tt>", 
-      "\\texttt{$1}"))
-  of rnSmiley: result = renderSmiley(d, n)
-  of rnLeaf: result = toRope(esc(d.target, n.text))
-  of rnContents: d.hasToc = true
-  of rnTitle: d.meta[metaTitle] = renderRstToOut(d, n.sons[0])
 
 proc checkForFalse(n: PNode): bool = 
   result = n.kind == nkIdent and IdentEq(n.ident, "false")
@@ -658,56 +275,52 @@ proc genSection(d: PDoc, kind: TSymKind) =
     "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return 
-  var title = toRope(sectionNames[kind])
+  var title = sectionNames[kind].toRope
   d.section[kind] = ropeFormatNamedVars(getConfigVar("doc.section"), [
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
-      toRope(ord(kind)), title, toRope(ord(kind) + 50), d.section[kind]])
+      ord(kind).toRope, title, toRope(ord(kind) + 50), d.section[kind]])
   d.toc[kind] = ropeFormatNamedVars(getConfigVar("doc.section.toc"), [
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
-      toRope(ord(kind)), title, toRope(ord(kind) + 50), d.toc[kind]])
+      ord(kind).toRope, title, toRope(ord(kind) + 50), d.toc[kind]])
 
 proc genOutFile(d: PDoc): PRope = 
   var 
-    code, toc, title, content: PRope
+    code, content: PRope = nil
-    bodyname: string
+    title = ""
-    j: int
+  var j = 0
-  j = 0
+  var tmp = ""
-  toc = renderTocEntries(d, j, 1)
+  renderTocEntries(d[], j, 1, tmp)
-  code = nil
+  var toc = tmp.toRope
-  content = nil
-  title = nil
   for i in countup(low(TSymKind), high(TSymKind)): 
     genSection(d, i)
     app(toc, d.toc[i])
   if toc != nil:
     toc = ropeFormatNamedVars(getConfigVar("doc.toc"), ["content"], [toc])
   for i in countup(low(TSymKind), high(TSymKind)): app(code, d.section[i])
-  if d.meta[metaTitle] != nil: title = d.meta[metaTitle]
+  if d.meta[metaTitle].len != 0: title = d.meta[metaTitle]
-  else: title = toRope("Module " &
+  else: title = "Module " & extractFilename(changeFileExt(d.filename, ""))
-      extractFilename(changeFileExt(d.filename, "")))
+  
-  if d.hasToc: bodyname = "doc.body_toc"
+  let bodyname = if d.hasToc: "doc.body_toc" else: "doc.body_no_toc"
-  else: bodyname = "doc.body_no_toc"
   content = ropeFormatNamedVars(getConfigVar(bodyname), ["title", 
       "tableofcontents", "moduledesc", "date", "time", "content"],
-      [title, toc, d.modDesc, toRope(getDateStr()), 
+      [title.toRope, toc, d.modDesc, toRope(getDateStr()), 
       toRope(getClockStr()), code])
   if optCompileOnly notin gGlobalOptions: 
+    # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(getConfigVar("doc.file"), ["title", 
         "tableofcontents", "moduledesc", "date", "time", 
         "content", "author", "version"], 
-        [title, toc, d.modDesc, toRope(getDateStr()), 
+        [title.toRope, toc, d.modDesc, toRope(getDateStr()), 
-                     toRope(getClockStr()), content, d.meta[metaAuthor], 
+                     toRope(getClockStr()), content, d.meta[metaAuthor].toRope, 
-                     d.meta[metaVersion]])
+                     d.meta[metaVersion].toRope])
   else: 
     code = content
   result = code
 
 proc generateIndex(d: PDoc) =
-  if d.theIndex != nil: 
+  if optGenIndex in gGlobalOptions:
-    sortIndex(d.theIndex)
+    writeIndexFile(d[], splitFile(options.outFile).dir / 
-    var content = newStringOfCap(2_000_000)
+                        splitFile(d.filename).name & indexExt)
-    renderRstToRst(d.indexFile, content)
-    writeFile(gIndexFile, content)
 
 proc writeOutput(d: PDoc, filename, outExt: string) = 
   var content = genOutFile(d)
@@ -716,11 +329,10 @@ proc writeOutput(d: PDoc, filename, outExt: string) =
   else:
     writeRope(content, getOutFile(filename, outExt))
 
-proc CommandDoc =
+proc CommandDoc*() =
   var ast = parseFile(addFileExt(gProjectFull, nimExt))
   if ast == nil: return 
-  var d = newDocumentor(gProjectFull)
+  var d = newDocumentor(gProjectFull, options.gConfigVars)
-  initIndexFile(d)
   d.hasToc = true
   generateDoc(d, ast)
   writeOutput(d, gProjectFull, HtmlExt)
@@ -728,74 +340,28 @@ proc CommandDoc =
 
 proc CommandRstAux(filename, outExt: string) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen)
+  var d = newDocumentor(filen, options.gConfigVars)
-  initIndexFile(d)
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})
-  d.modDesc = renderRstToOut(d, rst)
+  d.modDesc = newMutableRope(30_000)
+  renderRstToOut(d[], rst, d.modDesc.data)
+  freezeMutableRope(d.modDesc)
   writeOutput(d, filename, outExt)
   generateIndex(d)
 
-proc CommandRst2Html =
+proc CommandRst2Html*() =
   CommandRstAux(gProjectFull, HtmlExt)
 
-proc CommandRst2TeX =
+proc CommandRst2TeX*() =
   splitter = "\\-"
   CommandRstAux(gProjectFull, TexExt)
 
-# ---------- forum ---------------------------------------------------------
+proc CommandBuildIndex*() =
-
+  var content = mergeIndexes(gProjectFull).toRope
-proc setupConfig*() =
-  msgs.gErrorMax = 1000_000
-  setConfigVar("split.item.toc", "20")
-  setConfigVar("doc.section", """
-<div class="section" id="$sectionID">
-<h1><a class="toc-backref" href="#$sectionTitleID">$sectionTitle</a></h1>
-<dl class="item">
-$content
-</dl></div>
-""")
-  setConfigVar("doc.section.toc", """
-<li>
-  <a class="reference" href="#$sectionID" id="$sectionTitleID">$sectionTitle</a>
-  <ul class="simple">
-    $content
-  </ul>
-</li>
-""")
-  setConfigVar("doc.item", """
-<dt id="$itemID"><pre>$header</pre></dt>
-<dd>
-$desc
-</dd>
-""")
-  setConfigVar("doc.item.toc", """
-  <li><a class="reference" href="#$itemID">$name</a></li>
-""")
-  setConfigVar("doc.toc", """
-<div class="navigation" id="navigation">
-<ul class="simple">
-$content
-</ul>
-</div>""")
-  setConfigVar("doc.body_toc", """
-$tableofcontents
-<div class="content" id="content">
-$moduledesc
-$content
-</div>
-""")
-  setConfigVar("doc.body_no_toc", "$moduledesc $content")
-  setConfigVar("doc.file", "$content")
-
-proc rstToHtml*(s: string, options: TRstParseOptions): string =
-  ## exported for *nimforum*.
-  const filen = "input"
-  var d = newDocumentor(filen)
-  d.options = options
-  var dummyHasToc = false
-  var rst = rstParse(s, filen, 0, 1, dummyHasToc, options)
-  d.modDesc = renderRstToOut(d, rst)
-  let res = genOutFile(d)
-  result = res.ropeToStr
   
+  let code = ropeFormatNamedVars(getConfigVar("doc.file"), ["title", 
+      "tableofcontents", "moduledesc", "date", "time", 
+      "content", "author", "version"], 
+      ["Index".toRope, nil, nil, toRope(getDateStr()), 
+                   toRope(getClockStr()), content, nil, nil])
+  writeRope(code, getOutFile("theindex", HtmlExt))

compiler/main.nim

@@ -242,6 +242,10 @@ proc MainCommand =
     LoadConfigs(DocTexConfig)
     wantMainModule()
     CommandRst2TeX()
+  of "buildindex":
+    gCmd = cmdDoc
+    LoadConfigs(DocConfig)
+    CommandBuildIndex()
   of "gendepend": 
     gCmd = cmdGenDepend
     wantMainModule()

compiler/options.nim

@@ -53,7 +53,8 @@ type                          # please make sure we have under 32 options
     optDef,                   # ideTools: 'def'
     optThreadAnalysis,        # thread analysis pass
     optTaintMode,             # taint mode turned on
-    optTlsEmulation           # thread var emulation turned on
+    optTlsEmulation,          # thread var emulation turned on
+    optGenIndex               # generate index file for documentation
 
   TGlobalOptions* = set[TGlobalOption]
   TCommands* = enum           # Nimrod's commands
@@ -84,7 +85,6 @@ var
   gExitcode*: int8
   searchPaths*: TLinkedList
   outFile*: string = ""
-  gIndexFile*: string = ""
   gCmd*: TCommands = cmdNone  # the command
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors

compiler/ropes.nim

@@ -109,6 +109,9 @@ proc newMutableRope*(capacity = 30): PRope =
   new(result)
   result.data = newStringOfCap(capacity)
 
+proc freezeMutableRope*(r: PRope) {.inline.} =
+  r.length = r.data.len
+
 var 
   cache: array[0..2048 -1, PRope]
 

doc/advopt.txt

@@ -4,6 +4,7 @@ Advanced commands:
   //compileToOC, objc       compile project to Objective C code
   //rst2html                convert a reStructuredText file to HTML
   //rst2tex                 convert a reStructuredText file to TeX
+  //buildIndex              build an index for the whole documentation
   //run                     run the project (with Tiny C backend; buggy!)
   //pretty                  pretty print the inputfile
   //genDepend               generate a DOT file containing the
@@ -57,7 +58,7 @@ Advanced options:
   --skipParentCfg           do not read the parent dirs' configuration files
   --skipProjCfg             do not read the project's configuration file
   --gc:refc|boehm|none      use Nimrod's native GC|Boehm GC|no GC
-  --index:FILE              use FILE to generate a documentation index file
+  --index:on|off            turn index file generation on|off
   --putenv:key=value        set an environment variable
   --listCmd                 list the commands used to execute external programs
   --parallelBuild=0|1|...   perform a parallel build

koch.nim

@@ -148,7 +148,8 @@ proc boot(args: string) =
 const
   cleanExt = [
     ".ppu", ".o", ".obj", ".dcu", ".~pas", ".~inc", ".~dsk", ".~dpr",
-    ".map", ".tds", ".err", ".bak", ".pyc", ".exe", ".rod", ".pdb", ".idb"
+    ".map", ".tds", ".err", ".bak", ".pyc", ".exe", ".rod", ".pdb", ".idb",
+    ".idx"
   ]
   ignore = [
     ".bzrignore", "nimrod", "nimrod.exe", "koch", "koch.exe"

packages/docutils/rst.nim

@@ -38,7 +38,8 @@ type
     meGeneralParseError,
     meInvalidDirective,
     mwRedefinitionOfLabel,
-    mwUnknownSubstitution
+    mwUnknownSubstitution,
+    mwUnsupportedLanguage
   
   TMsgHandler* = proc (filename: string, line, col: int, msgKind: TMsgKind,
                        arg: string) ## what to do in case of an error
@@ -53,7 +54,8 @@ const
     meGeneralParseError: "general parse error",
     meInvalidDirective: "invalid directive: '$1'",
     mwRedefinitionOfLabel: "redefinition of label '$1'", 
-    mwUnknownSubstitution: "unknown substitution '$1'"
+    mwUnknownSubstitution: "unknown substitution '$1'",
+    mwUnsupportedLanguage: "language '$1' not supported"
   ]
 
 proc rstnodeToRefname*(n: PRstNode): string
@@ -281,17 +283,6 @@ type
 
   EParseError* = object of EInvalidValue
 
-when false:
-  proc tokInfo(p: TRstParser, tok: TToken): TLineInfo = 
-    result = newLineInfo(p.filename, p.line + tok.line, p.col + tok.col)
-
-  proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) = 
-    GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, arg)
-
-  proc rstMessage(p: TRstParser, msgKind: TMsgKind) = 
-    GlobalError(tokInfo(p, p.tok[p.idx]), msgKind, p.tok[p.idx].symbol)
-
-
 proc whichMsgClass*(k: TMsgKind): TMsgClass =
   ## returns which message class `k` belongs to.
   case ($k)[1]

packages/docutils/rstgen.nim

@@ -9,13 +9,60 @@
 
 ## This module implements a generator of HTML/Latex from `reStructuredText`:idx.
 
-import strutils, strtabs, rstast
+import strutils, os, hashes, strtabs, rstast, rst, highlite
+
+const
+  HtmlExt = "html"
+  IndexExt* = ".idx"
 
 type
   TOutputTarget* = enum ## which document type to generate
     outHtml,            # output is HTML
     outLatex            # output is Latex
   
+  TTocEntry{.final.} = object 
+    n*: PRstNode
+    refname*, header*: string
+
+  TMetaEnum* = enum 
+    metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion
+    
+  TRstGenerator* = object of TObject
+    target*: TOutputTarget
+    config*: PStringTable
+    splitAfter*: int          # split too long entries in the TOC
+    tocPart*: seq[TTocEntry]
+    hasToc*: bool
+    theIndex: string
+    options*: TRstParseOptions
+    findFile*: TFindFileHandler
+    msgHandler*: TMsgHandler
+    filename*: string
+    meta*: array[TMetaEnum, string]
+  
+  PDoc = var TRstGenerator
+
+proc initRstGenerator*(g: var TRstGenerator, target: TOutputTarget,
+                       config: PStringTable, filename: string,
+                       options: TRstParseOptions,
+                       findFile: TFindFileHandler,
+                       msgHandler: TMsgHandler) =
+  g.config = config
+  g.target = target
+  g.tocPart = @[]
+  g.filename = filename
+  g.splitAfter = 20
+  g.theIndex = ""
+  g.options = options
+  g.findFile = findFile
+  g.msgHandler = msgHandler
+  
+  let s = config["split.item.toc"]
+  if s != "": g.splitAfter = parseInt(s)
+  for i in low(g.meta)..high(g.meta): g.meta[i] = ""
+
+proc writeIndexFile*(g: var TRstGenerator, outfile: string) =
+  if g.theIndex.len > 0: writeFile(outfile, g.theIndex)
   
 proc addXmlChar(dest: var string, c: Char) = 
   case c
@@ -85,3 +132,562 @@ proc esc*(target: TOutputTarget, s: string, splitAfter = -1): string =
   else: 
     for i in countup(0, len(s) - 1): escChar(target, result, s[i])
 
+
+proc disp(target: TOutputTarget, xml, tex: string): string =
+  if target != outLatex: result = xml 
+  else: result = tex
+  
+proc dispF(target: TOutputTarget, xml, tex: string, 
+           args: openArray[string]): string = 
+  if target != outLatex: result = xml % args 
+  else: result = tex % args
+  
+proc dispA(target: TOutputTarget, dest: var string, 
+           xml, tex: string, args: openarray[string]) =
+  if target != outLatex: addf(dest, xml, args)
+  else: addf(dest, tex, args)
+  
+proc renderRstToOut*(d: PDoc, n: PRstNode, result: var string)
+
+proc renderAux(d: PDoc, n: PRstNode, result: var string) = 
+  for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], result)
+
+proc renderAux(d: PDoc, n: PRstNode, frmtA, frmtB: string, result: var string) = 
+  var tmp = ""
+  for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], tmp)
+  if d.target != outLatex:
+    result.addf(frmtA, [tmp])
+  else:
+    result.addf(frmtB, [tmp])
+
+# ---------------- index handling --------------------------------------------
+
+proc setIndexTerm*(d: PDoc, id, term: string) =
+  d.theIndex.add(term)
+  d.theIndex.add('\t')
+  let htmlFile = changeFileExt(extractFilename(d.filename), HtmlExt)
+  d.theIndex.add(htmlFile)
+  d.theIndex.add('#')
+  d.theIndex.add(id)
+  d.theIndex.add("\n")
+
+proc hash(n: PRstNode): int =
+  if n.kind == rnLeaf:
+    result = hash(n.text)
+  elif n.len > 0:
+    result = hash(n.sons[0])
+    for i in 1 .. <len(n):
+      result = result !& hash(n.sons[i])
+    result = !$result
+
+proc renderIndexTerm(d: PDoc, n: PRstNode, result: var string) =
+  let id = rstnodeToRefname(n) & '_' & $abs(hash(n))
+  var term = ""
+  renderAux(d, n, term)
+  setIndexTerm(d, id, term)
+  dispA(d.target, result, "<span id=\"$1\">$2</span>", "$2\\label{$1}", 
+        [id, term])
+
+type
+  TIndexEntry {.pure, final.} = object
+    keyword: string
+    link: string
+
+proc cmp(a, b: TIndexEntry): int =
+  result = cmpIgnoreStyle(a.keyword, b.keyword)
+
+proc `<-`(a: var TIndexEntry, b: TIndexEntry) =
+  shallowCopy a.keyword, b.keyword
+  shallowCopy a.link, b.link
+
+proc sortIndex(a: var openArray[TIndexEntry]) =
+  # we use shellsort here; fast and simple
+  let N = len(a)
+  var h = 1
+  while true:
+    h = 3 * h + 1
+    if h > N: break
+  while true:
+    h = h div 3
+    for i in countup(h, N - 1):
+      var v: TIndexEntry
+      v <- a[i]
+      var j = i
+      while cmp(a[j-h], v) >= 0:
+        a[j] <- a[j-h]
+        j = j-h
+        if j < h: break
+      a[j] <- v
+    if h == 1: break
+
+proc mergeIndexes*(dir: string): string =
+  ## merges all index files in `dir` and returns the generated index as HTML.
+  ## The result is no full HTML for flexibility.
+  var a: seq[TIndexEntry]
+  newSeq(a, 15_000)
+  setLen(a, 0)
+  var L = 0
+  for kind, path in walkDir(dir):
+    if kind == pcFile and path.endsWith(IndexExt):
+      for line in lines(path):
+        let s = line.find('\t')
+        if s < 0: continue
+        setLen(a, L+1)
+        a[L].keyword = line.substr(0, s-1)
+        a[L].link = line.substr(s+1)
+        inc L
+  sortIndex(a)
+  result = ""
+  var i = 0
+  while i < L:
+    result.addf("<dt><span>$1</span></dt><ul class=\"simple\"><dd>\n", 
+                a[i].keyword)
+    var j = i
+    while j < L and a[i].keyword == a[j].keyword:
+      result.addf(
+        "<li><a class=\"reference external\" href=\"$1\">$1</a></li>\n", 
+        a[j].link)
+      inc j
+    result.add("</ul></dd>\n")
+    i = j
+  
+# ----------------------------------------------------------------------------      
+  
+proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = 
+  var tmp = ""
+  for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
+  var refname = rstnodeToRefname(n)
+  if d.hasToc:
+    var length = len(d.tocPart)
+    setlen(d.tocPart, length + 1)
+    d.tocPart[length].refname = refname
+    d.tocPart[length].n = n
+    d.tocPart[length].header = tmp
+    
+    dispA(d.target, result,
+        "<h$1><a class=\"toc-backref\" id=\"$2\" href=\"#$2_toc\">$3</a></h$1>", 
+        "\\rsth$4{$3}\\label{$2}\n", [$n.level, 
+        d.tocPart[length].refname, tmp, 
+        $chr(n.level - 1 + ord('A'))])
+  else:
+    dispA(d.target, result, "<h$1 id=\"$2\">$3</h$1>", 
+                            "\\rsth$4{$3}\\label{$2}\n", [
+        $n.level, refname, tmp, 
+        $chr(n.level - 1 + ord('A'))])
+  
+proc renderOverline(d: PDoc, n: PRstNode, result: var string) = 
+  if d.meta[metaTitle].len == 0:
+    for i in countup(0, len(n)-1):
+      renderRstToOut(d, n.sons[i], d.meta[metaTitle])
+  elif d.meta[metaSubtitle].len == 0:
+    for i in countup(0, len(n)-1):
+      renderRstToOut(d, n.sons[i], d.meta[metaSubtitle])
+  else:
+    var tmp = ""
+    for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
+    dispA(d.target, result, "<h$1 id=\"$2\"><center>$3</center></h$1>", 
+                   "\\rstov$4{$3}\\label{$2}\n", [$n.level,
+        rstnodeToRefname(n), tmp, $chr(n.level - 1 + ord('A'))])
+  
+
+proc renderTocEntry(d: PDoc, e: TTocEntry, result: var string) = 
+  dispA(d.target, result,
+    "<li><a class=\"reference\" id=\"$1_toc\" href=\"#$1\">$2</a></li>\n", 
+    "\\item\\label{$1_toc} $2\\ref{$1}\n", [e.refname, e.header])
+
+proc renderTocEntries*(d: PDoc, j: var int, lvl: int, result: var string) = 
+  var tmp = ""
+  while j <= high(d.tocPart): 
+    var a = abs(d.tocPart[j].n.level)
+    if a == lvl:
+      renderTocEntry(d, d.tocPart[j], tmp)
+      inc(j)
+    elif a > lvl:
+      renderTocEntries(d, j, a, tmp)
+    else:
+      break
+  if lvl > 1:
+    dispA(d.target, result, "<ul class=\"simple\">$1</ul>", 
+                            "\\begin{enumerate}$1\\end{enumerate}", [tmp])
+  else:
+    result.add(tmp)
+  
+proc renderImage(d: PDoc, n: PRstNode, result: var string) = 
+  var options = ""
+  var s = getFieldValue(n, "scale")
+  if s != "": dispA(d.target, options, " scale=\"$1\"", " scale=$1", [strip(s)])
+  
+  s = getFieldValue(n, "height")
+  if s != "": dispA(d.target, options, " height=\"$1\"", " height=$1", [strip(s)])
+  
+  s = getFieldValue(n, "width")
+  if s != "": dispA(d.target, options, " width=\"$1\"", " width=$1", [strip(s)])
+  
+  s = getFieldValue(n, "alt")
+  if s != "": dispA(d.target, options, " alt=\"$1\"", "", [strip(s)])
+  
+  s = getFieldValue(n, "align")
+  if s != "": dispA(d.target, options, " align=\"$1\"", "", [strip(s)])
+  
+  if options.len > 0: options = dispF(d.target, "$1", "[$1]", [options])
+  
+  dispA(d.target, result, "<img src=\"$1\"$2 />", "\\includegraphics$2{$1}", 
+                 [getArgument(n), options])
+  if len(n) >= 3: renderRstToOut(d, n.sons[2], result)
+  
+proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
+  dispA(d.target, result,
+    """<img src="images/smilies/$1.gif" width="15" 
+        height="17" hspace="2" vspace="2" />""",
+    "\\includegraphics{$1}", [n.text])
+  
+proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
+  if n.sons[2] == nil: return
+  var m = n.sons[2].sons[0]
+  assert m.kind == rnLeaf
+  var langstr = strip(getArgument(n))
+  var lang: TSourceLanguage
+  if langstr == "":
+    lang = langNimrod         # default language
+  else:
+    lang = getSourceLanguage(langstr)
+  
+  dispA(d.target, result, "<pre>", "\\begin{rstpre}\n")
+  if lang == langNone:
+    d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, langstr)
+    result.add(m.text)
+  else:
+    var g: TGeneralTokenizer
+    initGeneralTokenizer(g, m.text)
+    while true: 
+      getNextToken(g, lang)
+      case g.kind
+      of gtEof: break 
+      of gtNone, gtWhitespace: 
+        add(result, substr(m.text, g.start, g.length + g.start - 1))
+      else:
+        dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}", [
+          esc(d.target, substr(m.text, g.start, g.length+g.start-1)),
+          tokenClassToStr[g.kind]])
+    deinitGeneralTokenizer(g)
+  dispA(d.target, result, "</pre>", "\n\\end{rstpre}\n")
+  
+proc renderContainer(d: PDoc, n: PRstNode, result: var string) = 
+  var tmp = ""
+  renderRstToOut(d, n.sons[2], tmp)
+  var arg = strip(getArgument(n))
+  if arg == "": 
+    dispA(d.target, result, "<div>$1</div>", "$1", [tmp])
+  else:
+    dispA(d.target, result, "<div class=\"$1\">$2</div>", "$2", [arg, tmp])
+  
+proc texColumns(n: PRstNode): string = 
+  result = ""
+  for i in countup(1, len(n)): add(result, "|X")
+  
+proc renderField(d: PDoc, n: PRstNode, result: var string) = 
+  var b = false
+  if d.target == outLatex: 
+    var fieldname = addNodes(n.sons[0])
+    var fieldval = esc(d.target, strip(addNodes(n.sons[1])))
+    if cmpIgnoreStyle(fieldname, "author") == 0:
+      if d.meta[metaAuthor].len == 0:
+        d.meta[metaAuthor] = fieldval
+        b = true
+    elif cmpIgnoreStyle(fieldName, "version") == 0: 
+      if d.meta[metaVersion].len == 0:
+        d.meta[metaVersion] = fieldval
+        b = true
+  if not b:
+    renderAux(d, n, "<tr>$1</tr>\n", "$1", result)
+  
+proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
+  if n == nil: return
+  case n.kind
+  of rnInner: renderAux(d, n, result)
+  of rnHeadline: renderHeadline(d, n, result)
+  of rnOverline: renderOverline(d, n, result)
+  of rnTransition: renderAux(d, n, "<hr />\n", "\\hrule\n", result)
+  of rnParagraph: renderAux(d, n, "<p>$1</p>\n", "$1\n\n", result)
+  of rnBulletList:
+    renderAux(d, n, "<ul class=\"simple\">$1</ul>\n",
+                    "\\begin{itemize}$1\\end{itemize}\n", result)
+  of rnBulletItem, rnEnumItem:
+    renderAux(d, n, "<li>$1</li>\n", "\\item $1\n", result)
+  of rnEnumList:
+    renderAux(d, n, "<ol class=\"simple\">$1</ol>\n",
+                    "\\begin{enumerate}$1\\end{enumerate}\n", result)
+  of rnDefList:
+    renderAux(d, n, "<dl class=\"docutils\">$1</dl>\n",
+                       "\\begin{description}$1\\end{description}\n", result)
+  of rnDefItem: renderAux(d, n, result)
+  of rnDefName: renderAux(d, n, "<dt>$1</dt>\n", "\\item[$1] ", result)
+  of rnDefBody: renderAux(d, n, "<dd>$1</dd>\n", "$1\n", result)
+  of rnFieldList:
+    var tmp = ""
+    for i in countup(0, len(n) - 1): 
+      renderRstToOut(d, n.sons[i], tmp)
+    if tmp.len != 0: 
+      dispA(d.target, result,
+          "<table class=\"docinfo\" frame=\"void\" rules=\"none\">" &
+          "<col class=\"docinfo-name\" />" &
+          "<col class=\"docinfo-content\" />" & 
+          "<tbody valign=\"top\">$1" &
+          "</tbody></table>", 
+          "\\begin{description}$1\\end{description}\n", 
+          [tmp])
+  of rnField: renderField(d, n, result)
+  of rnFieldName: 
+    renderAux(d, n, "<th class=\"docinfo-name\">$1:</th>", "\\item[$1:]", result)
+  of rnFieldBody: 
+    renderAux(d, n, "<td>$1</td>", " $1\n", result)
+  of rnIndex: 
+    renderRstToOut(d, n.sons[2], result)
+  of rnOptionList: 
+    renderAux(d, n, "<table frame=\"void\">$1</table>", 
+      "\\begin{description}\n$1\\end{description}\n", result)
+  of rnOptionListItem: 
+    renderAux(d, n, "<tr>$1</tr>\n", "$1", result)
+  of rnOptionGroup: 
+    renderAux(d, n, "<th align=\"left\">$1</th>", "\\item[$1]", result)
+  of rnDescription: 
+    renderAux(d, n, "<td align=\"left\">$1</td>\n", " $1\n", result)
+  of rnOption, rnOptionString, rnOptionArgument: 
+    doAssert false, "renderRstToOut"
+  of rnLiteralBlock:
+    renderAux(d, n, "<pre>$1</pre>\n", 
+                    "\\begin{rstpre}\n$1\n\\end{rstpre}\n", result)
+  of rnQuotedLiteralBlock: 
+    doAssert false, "renderRstToOut"
+  of rnLineBlock: 
+    renderAux(d, n, "<p>$1</p>", "$1\n\n", result)
+  of rnLineBlockItem: 
+    renderAux(d, n, "$1<br />", "$1\\\\\n", result)
+  of rnBlockQuote: 
+    renderAux(d, n, "<blockquote><p>$1</p></blockquote>\n", 
+                    "\\begin{quote}$1\\end{quote}\n", result)
+  of rnTable, rnGridTable: 
+    renderAux(d, n, 
+      "<table border=\"1\" class=\"docutils\">$1</table>", 
+      "\\begin{table}\\begin{rsttab}{" &
+        texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result)
+  of rnTableRow: 
+    if len(n) >= 1:
+      if d.target == outLatex:
+        var tmp = ""
+        renderRstToOut(d, n.sons[0], tmp)
+        for i in countup(1, len(n) - 1):
+          result.add(" & ")
+          renderRstToOut(d, n.sons[i], result)
+        result.add("\\\\\n\\hline\n")
+      else:
+        result.add("<tr>")
+        renderAux(d, n, result)
+        result.add("</tr>\n")
+  of rnTableDataCell: 
+    renderAux(d, n, "<td>$1</td>", "$1", result)
+  of rnTableHeaderCell: 
+    renderAux(d, n, "<th>$1</th>", "\\textbf{$1}", result)
+  of rnLabel: 
+    doAssert false, "renderRstToOut" # used for footnotes and other
+  of rnFootnote: 
+    doAssert false, "renderRstToOut" # a footnote
+  of rnCitation: 
+    doAssert false, "renderRstToOut" # similar to footnote
+  of rnRef: 
+    var tmp = ""
+    renderAux(d, n, tmp)
+    dispA(d.target, result, "<a class=\"reference external\" href=\"#$2\">$1</a>", 
+                            "$1\\ref{$2}", [tmp, rstnodeToRefname(n)])
+  of rnStandaloneHyperlink: 
+    renderAux(d, n, 
+      "<a class=\"reference external\" href=\"$1\">$1</a>", 
+      "\\href{$1}{$1}", result)
+  of rnHyperlink:
+    var tmp0 = ""
+    var tmp1 = ""
+    renderRstToOut(d, n.sons[0], tmp0)
+    renderRstToOut(d, n.sons[1], tmp1)
+    dispA(d.target, result, "<a class=\"reference external\" href=\"$2\">$1</a>", 
+                   "\\href{$2}{$1}", 
+                   [tmp0, tmp1])
+  of rnDirArg, rnRaw: renderAux(d, n, result)
+  of rnRawHtml:
+    if d.target != outLatex:
+      result.add addNodes(lastSon(n))
+  of rnRawLatex:
+    if d.target == outLatex:
+      result.add addNodes(lastSon(n))
+      
+  of rnImage, rnFigure: renderImage(d, n, result)
+  of rnCodeBlock: renderCodeBlock(d, n, result)
+  of rnContainer: renderContainer(d, n, result)
+  of rnSubstitutionReferences, rnSubstitutionDef: 
+    renderAux(d, n, "|$1|", "|$1|", result)
+  of rnDirective:
+    renderAux(d, n, "", "", result)
+  of rnGeneralRole:
+    var tmp0 = ""
+    var tmp1 = ""
+    renderRstToOut(d, n.sons[0], tmp0)
+    renderRstToOut(d, n.sons[1], tmp1)
+    dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}",
+          [tmp0, tmp1])
+  of rnSub: renderAux(d, n, "<sub>$1</sub>", "\\rstsub{$1}", result)
+  of rnSup: renderAux(d, n, "<sup>$1</sup>", "\\rstsup{$1}", result)
+  of rnEmphasis: renderAux(d, n, "<em>$1</em>", "\\emph{$1}", result)
+  of rnStrongEmphasis:
+    renderAux(d, n, "<strong>$1</strong>", "\\textbf{$1}", result)
+  of rnTripleEmphasis:
+    renderAux(d, n, "<strong><em>$1</em></strong>", 
+                    "\\textbf{emph{$1}}", result)
+  of rnInterpretedText:
+    renderAux(d, n, "<cite>$1</cite>", "\\emph{$1}", result)
+  of rnIdx:
+    renderIndexTerm(d, n, result)
+  of rnInlineLiteral: 
+    renderAux(d, n, 
+      "<tt class=\"docutils literal\"><span class=\"pre\">$1</span></tt>", 
+      "\\texttt{$1}", result)
+  of rnSmiley: renderSmiley(d, n, result)
+  of rnLeaf: result.add(esc(d.target, n.text))
+  of rnContents: d.hasToc = true
+  of rnTitle:
+    d.meta[metaTitle] = ""
+    renderRstToOut(d, n.sons[0], d.meta[metaTitle])
+
+# -----------------------------------------------------------------------------
+
+proc getVarIdx(varnames: openarray[string], id: string): int = 
+  for i in countup(0, high(varnames)): 
+    if cmpIgnoreStyle(varnames[i], id) == 0: 
+      return i
+  result = -1
+
+proc formatNamedVars*(frmt: string, varnames: openarray[string], 
+                      varvalues: openarray[string]): string = 
+  var i = 0
+  var L = len(frmt)
+  result = ""
+  var num = 0
+  while i < L: 
+    if frmt[i] == '$': 
+      inc(i)                  # skip '$'
+      case frmt[i]
+      of '#': 
+        add(result, varvalues[num])
+        inc(num)
+        inc(i)
+      of '$': 
+        add(result, "$")
+        inc(i)
+      of '0'..'9': 
+        var j = 0
+        while true: 
+          j = (j * 10) + Ord(frmt[i]) - ord('0')
+          inc(i)
+          if i > L-1 or frmt[i] notin {'0'..'9'}: break 
+        if j > high(varvalues) + 1:
+          raise newException(EInvalidValue, "invalid index: " & $j)
+        num = j
+        add(result, varvalues[j - 1])
+      of 'A'..'Z', 'a'..'z', '\x80'..'\xFF': 
+        var id = ""
+        while true: 
+          add(id, frmt[i])
+          inc(i)
+          if frmt[i] notin {'A'..'Z', '_', 'a'..'z', '\x80'..'\xFF'}: break 
+        var idx = getVarIdx(varnames, id)
+        if idx >= 0: 
+          add(result, varvalues[idx])
+        else:
+          raise newException(EInvalidValue, "unknown substitution var: " & id)
+      of '{': 
+        var id = ""
+        inc(i)
+        while frmt[i] != '}': 
+          if frmt[i] == '\0': 
+            raise newException(EInvalidValue, "'}' expected")
+          add(id, frmt[i])
+          inc(i)
+        inc(i)                # skip }
+                              # search for the variable:
+        var idx = getVarIdx(varnames, id)
+        if idx >= 0: add(result, varvalues[idx])
+        else: 
+          raise newException(EInvalidValue, "unknown substitution var: " & id)
+      else:
+        raise newException(EInvalidValue, "unknown substitution: $" & $frmt[i])
+    var start = i
+    while i < L: 
+      if frmt[i] != '$': inc(i)
+      else: break
+    if i-1 >= start: add(result, substr(frmt, start, i - 1))
+
+
+proc defaultConfig*(): PStringTable =
+  ## creates a default configuration for HTML generation.
+  result = newStringTable(modeStyleInsensitive)
+  
+  template setConfigVar(key, val: expr) =
+    result[key] = val
+  
+  setConfigVar("split.item.toc", "20")
+  setConfigVar("doc.section", """
+<div class="section" id="$sectionID">
+<h1><a class="toc-backref" href="#$sectionTitleID">$sectionTitle</a></h1>
+<dl class="item">
+$content
+</dl></div>
+""")
+  setConfigVar("doc.section.toc", """
+<li>
+  <a class="reference" href="#$sectionID" id="$sectionTitleID">$sectionTitle</a>
+  <ul class="simple">
+    $content
+  </ul>
+</li>
+""")
+  setConfigVar("doc.item", """
+<dt id="$itemID"><pre>$header</pre></dt>
+<dd>
+$desc
+</dd>
+""")
+  setConfigVar("doc.item.toc", """
+  <li><a class="reference" href="#$itemID">$name</a></li>
+""")
+  setConfigVar("doc.toc", """
+<div class="navigation" id="navigation">
+<ul class="simple">
+$content
+</ul>
+</div>""")
+  setConfigVar("doc.body_toc", """
+$tableofcontents
+<div class="content" id="content">
+$moduledesc
+$content
+</div>
+""")
+  setConfigVar("doc.body_no_toc", "$moduledesc $content")
+  setConfigVar("doc.file", "$content")
+
+# ---------- forum ---------------------------------------------------------
+
+proc rstToHtml*(s: string, options: TRstParseOptions, 
+                config: PStringTable): string =
+  ## exported for *nimforum*.
+  
+  proc myFindFile(filename: string): string = 
+    # we don't find any files in online mode:
+    result = ""
+
+  const filen = "input"
+  var d: TRstGenerator
+  initRstGenerator(d, outHtml, config, filen, options, myFindFile, nil)
+  var dummyHasToc = false
+  var rst = rstParse(s, filen, 0, 1, dummyHasToc, options)
+  result = ""
+  renderRstToOut(d, rst, result)
+  

todo.txt

@@ -1,6 +1,7 @@
 version 0.9.0
 =============
 
+- docgen: markdown support
 - make templates hygienic by default
 - ``bind`` for overloaded symbols does not work apparently
 - ``=`` should be overloadable; requires specialization for ``=``

tools/nimweb.nim

@@ -162,15 +162,12 @@ proc Exec(cmd: string) =
 proc buildDoc(c: var TConfigData, destPath: string) =
   # call nim for the documentation:
   for d in items(c.doc):
-    Exec("nimrod rst2html $# -o:$# --index=$#/theindex $#" %
+    Exec("nimrod rst2html $# -o:$# --index:on $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"),
+      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
-       destpath, d])
   for d in items(c.srcdoc):
-    Exec("nimrod doc $# -o:$# --index=$#/theindex $#" %
+    Exec("nimrod doc $# -o:$# --index:on $#" %
-      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"),
+      [c.nimrodArgs, destPath / changeFileExt(splitFile(d).name, "html"), d])
-       destpath, d])
+  Exec("nimrod buildIndex -o:$1/theindex.html $1" % [destPath])
-  Exec("nimrod rst2html $1 -o:$2/theindex.html $2/theindex" %
-       [c.nimrodArgs, destPath])
 
 proc buildPdfDoc(c: var TConfigData, destPath: string) =
   if os.execShellCmd("pdflatex -version") != 0: