doc2tex: generate docs to Latex (#17997)

* `doc2tex`: generate docs to Latex

* address some comments

compiler/commands.nim

@@ -444,6 +444,7 @@ proc parseCommand*(command: string): Command =
   of "e": cmdNimscript
   of "doc0": cmdDoc0
   of "doc2", "doc": cmdDoc2
+  of "doc2tex": cmdDoc2tex
   of "rst2html": cmdRst2html
   of "rst2tex": cmdRst2tex
   of "jsondoc0": cmdJsondoc0

compiler/docgen.nim

@@ -169,14 +169,17 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
   else:
     result = getOutFile(conf, filename, ext)
 
-proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil): PDoc =
+proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
+
+proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
+                    outExt: string = HtmlExt, module: PSym = nil): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
+  initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
                    conf.configVars, filename.string,
                    {roSupportRawDirective, roSupportMarkdown,
                     roPreferMarkdown, roNimFile},
@@ -249,7 +252,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
 
 template dispA(conf: ConfigRef; dest: var string, xml, tex: string,
                args: openArray[string]) =
-  if conf.cmd != cmdRst2tex: dest.addf(xml, args)
+  if not conf.isLatexCmd: dest.addf(xml, args)
   else: dest.addf(tex, args)
 
 proc getVarIdx(varnames: openArray[string], id: string): int =
@@ -551,7 +554,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
         let id = $d.listingCounter
         dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
         var dest2 = ""
-        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        renderNimCode(dest2, code, d.target)
         dest.add dest2
         dest.add(d.config.getOrDefault"doc.listing_end" % id)
         return rsRunnable
@@ -821,7 +824,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
 
   inc(d.id)
   let
-    plainNameEsc = xmltree.escape(plainName.strip)
+    plainNameEsc = esc(d.target, plainName.strip)
+    uniqueName = if k in routineKinds: plainNameEsc else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
     plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -835,7 +839,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
   d.section[k].add(getConfigVar(d.conf, "doc.item") %
-    ["name", name, "header", result, "desc", comm, "itemID", $d.id,
+    ["name", name, "uniqueName", uniqueName,
+     "header", result, "desc", comm, "itemID", $d.id,
      "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
      "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc,
      "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc,
@@ -1199,7 +1204,8 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
     var shouldSort = i in routineKinds and groupedToc
     genSection(d, i, shouldSort)
     toc.add(d.toc[i])
-  if toc != "":
+  if toc != "" or d.target == outLatex:
+    # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set
     toc = getConfigVar(d.conf, "doc.toc") % ["content", toc]
   for i in TSymKind: code.add(d.section[i])
 
@@ -1217,7 +1223,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string =
         "\\\\\\vspace{0.5em}\\large $1", [d.meta[metaSubtitle]])
 
   var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
-  let bodyname = if d.hasToc and not d.isPureRst:
+  let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd:
                    groupsection.setLen 0
                    "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"

compiler/docgen2.nim

@@ -71,10 +71,15 @@ template myOpenImpl(ext: untyped) {.dirty.} =
 proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
+proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+  myOpenImpl(TexExt)
+
 proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
 
 const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
+const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
+                                 close = close)
 const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
                                   close = closeJson)
 

compiler/main.nim

@@ -69,12 +69,15 @@ proc commandCheck(graph: ModuleGraph) =
     writeRodFiles(graph)
 
 when not defined(leanCompiler):
-  proc commandDoc2(graph: ModuleGraph; json: bool) =
+  proc commandDoc2(graph: ModuleGraph; ext: string) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
-    if json: registerPass(graph, docgen2JsonPass)
-    else: registerPass(graph, docgen2Pass)
+    case ext:
+    of TexExt:  registerPass(graph, docgen2TexPass)
+    of JsonExt: registerPass(graph, docgen2JsonPass)
+    of HtmlExt: registerPass(graph, docgen2Pass)
+    else: doAssert false, $ext
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
@@ -249,7 +252,8 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      loadConfigs(DocConfig, cache, conf, graph.idgen)
+      let docConf = if conf.cmd == cmdDoc2tex: DocTexConfig else: DocConfig
+      loadConfigs(docConf, cache, conf, graph.idgen)
       defineSymbol(conf.symbols, "nimdoc")
       body
 
@@ -285,7 +289,7 @@ proc mainCommand*(graph: ModuleGraph) =
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
         #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-      commandDoc2(graph, false)
+      commandDoc2(graph, HtmlExt)
       if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
         commandBuildIndex(conf, $conf.outDir)
   of cmdRst2html:
@@ -299,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) =
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
       commandRst2Html(cache, conf)
-  of cmdRst2tex:
+  of cmdRst2tex, cmdDoc2tex:
     for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
                  warnLanguageXNotSupported,
                  warnFieldXNotSupported, warnRstStyle]:
@@ -307,10 +311,13 @@ proc mainCommand*(graph: ModuleGraph) =
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
-      loadConfigs(DocTexConfig, cache, conf, graph.idgen)
-      commandRst2TeX(cache, conf)
+      if conf.cmd == cmdRst2tex:
+        loadConfigs(DocTexConfig, cache, conf, graph.idgen)
+        commandRst2TeX(cache, conf)
+      else:
+        docLikeCmd commandDoc2(graph, TexExt)
   of cmdJsondoc0: docLikeCmd commandJson(cache, conf)
-  of cmdJsondoc: docLikeCmd commandDoc2(graph, true)
+  of cmdJsondoc: docLikeCmd commandDoc2(graph, JsonExt)
   of cmdCtags: docLikeCmd commandTags(cache, conf)
   of cmdBuildindex: docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of cmdGendepend: commandGenDepend(graph)

compiler/options.nim

@@ -144,6 +144,7 @@ type
     cmdNimscript # evaluate nimscript
     cmdDoc0
     cmdDoc2
+    cmdDoc2tex
     cmdRst2html # convert a reStructuredText file to HTML
     cmdRst2tex # convert a reStructuredText file to TeX
     cmdJsondoc0
@@ -160,7 +161,8 @@ type
 
 const
   cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun}
-  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex}
+  cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc,
+                 cmdCtags, cmdBuildindex}
 
 type
   TStringSeq* = seq[string]

compiler/renderverbatim.nim

@@ -1,8 +1,6 @@
 import strutils
-from xmltree import addEscaped
 
 import ast, options, msgs
-import packages/docutils/highlite
 
 const isDebug = false
 when isDebug:
@@ -131,22 +129,3 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
       lastNonemptyPos = result.len
   result.setLen lastNonemptyPos
 
-proc renderNimCode*(result: var string, code: string, isLatex = false) =
-  var toknizr: GeneralTokenizer
-  initGeneralTokenizer(toknizr, code)
-  var buf = ""
-  template append(kind, val) =
-    buf.setLen 0
-    buf.addEscaped(val)
-    let class = tokenClassToStr[kind]
-    if isLatex:
-      result.addf "\\span$1{$2}", [class, buf]
-    else:
-      result.addf  "<span class=\"$1\">$2</span>", [class, buf]
-  while true:
-    getNextToken(toknizr, langNim)
-    case toknizr.kind
-    of gtEof: break  # End Of File (or string)
-    else:
-      # TODO: avoid alloc; maybe toOpenArray
-      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))