fix #6583, fix #14376, index+search now generated for all projects, many bug fixes with nim doc (#14324)

* refs #6583 fix nim doc output * changelog * change default for outDir when unspecified * cleanups * --project implies --index
2020-05-25 04:25:40 -07:00 · 2020-05-25 04:25:40 -07:00 · 58282547f6
commit 58282547f6
parent cbfe9325c5
22 changed files with 294 additions and 161 deletions
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@ -440,7 +440,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
    expectArg(conf, switch, arg, pass, info)
    conf.docSeeSrcUrl = arg
  of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
  of "backend", "b":
    let backend = parseEnum(arg.normalize, TBackend.default)
    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
@ -485,7 +485,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
  of "forcebuild", "f":
    processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
  of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
  of "gc":
    expectArg(conf, switch, arg, pass, info)
    if pass in {passCmd2, passPP}:
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@ -17,11 +17,10 @@ import
  packages/docutils/rst, packages/docutils/rstgen,
  json, xmltree, cgi, trees, types,
  typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables
+  pathutils, trees, tables, nimpaths

 const
  exportSection = skField
-  htmldocsDir = RelativeDir"htmldocs"
  docCmdSkip = "skip"

 type
@ -51,7 +50,7 @@ type
    destFile*: AbsoluteFile
    thisDir*: AbsoluteDir
    exampleGroups: OrderedTable[string, ExampleGroup]
-    wroteCss*: bool
+    wroteSupportFiles*: bool

  PDoc* = ref TDocumentor ## Alias to type less.

@ -72,7 +71,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
  proc nimbleDir(): AbsoluteDir =
    getNimbleFile(conf, file2).parentDir.AbsoluteDir
  case conf.docRoot:
-  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+  of docRootDefault:
    result = getRelativePathFromConfigPath(conf, file)
    let dir = nimbleDir()
    if not dir.isEmpty:
@ -88,9 +87,12 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
    result = getRelativePathFromConfigPath(conf, file)
    if result.isEmpty: bail()
  elif conf.docRoot.len > 0:
-    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
-    doAssert conf.docRoot.existsDir, conf.docRoot
-    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+    # we're (currently) requiring `isAbsolute` to avoid confusion when passing
+    # a relative path (would it be relative wrt $PWD or to projectfile)
+    conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
+    conf.globalAssert conf.docRoot.existsDir, arg=conf.docRoot
+    # needed because `canonicalizePath` called on `file`
+    result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
  else:
    bail()
  if isAbsolute(result.string):
@ -1125,11 +1127,8 @@ proc genSection(d: PDoc, kind: TSymKind) =
      "sectionid", "sectionTitle", "sectionTitleID", "content"], [
      ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])

-const nimdocOutCss = "nimdoc.out.css"
-  # `out` to make it easier to use with gitignore in user's repos
-
-proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
-  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
+proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): Rope =
+  rope($relativeTo(outDir / linkto, destFile.splitFile().dir, '/'))

 proc genOutFile(d: PDoc): Rope =
  var
@ -1160,15 +1159,17 @@ proc genOutFile(d: PDoc): Rope =
                 elif d.hasToc: "doc.body_toc"
                 else: "doc.body_no_toc"
  content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref"],
      [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile, theindexFname.RelativeFile)])
  if optCompileOnly notin d.conf.globalOptions:
    # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
    code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
        "content", "author", "version", "analytics", "deprecationMsg"],
-        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        [relLink(d.conf.outDir, d.destFile, nimdocOutCss.RelativeFile),
+        relLink(d.conf.outDir, d.destFile, docHackJsFname.RelativeFile),
+        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
        content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
  else:
    code = content
@ -1203,11 +1204,13 @@ proc writeOutput*(d: PDoc, useWarning = false) =
    if not writeRope(content, outfile):
      rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
        outfile.string)
-    elif not d.wroteCss:
-      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
-      let cssDest = $dir / nimdocOutCss
-      copyFile(cssSource, cssDest)
-      d.wroteCss = true
+    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
+      let nimr = $d.conf.getPrefixDir()
+      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      if optGenIndex in d.conf.globalOptions:
+        let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
+        copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
+      d.wroteSupportFiles = true

 proc writeOutputJson*(d: PDoc, useWarning = false) =
  runAllExamples(d)
@ -1234,6 +1237,8 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
 proc handleDocOutputOptions*(conf: ConfigRef) =
  if optWholeProject in conf.globalOptions:
    # Backward compatibility with previous versions
+    # xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
+    # it'd write to `sub/bar.html/main.html`
    conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)

 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
@ -1308,20 +1313,23 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
    if not writeRope(content, filename):
      rawMessage(conf, errCannotOpenFile, filename.string)

-proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
-  var content = mergeIndexes(conf.projectFull.string).rope
+proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  var content = mergeIndexes(dir).rope

-  var outFile = RelativeFile"theindex"
-  if conf.outFile != RelativeFile"":
-    outFile = conf.outFile
+  var outFile = outFile
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
  let filename = getOutFile(conf, outFile, HtmlExt)

  let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+      "nimdoccss", "dochackjs",
+      "title", "tableofcontents", "moduledesc", "date", "time",
      "content", "author", "version", "analytics"],
-      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      rope"Index", nil, nil, rope(getDateStr()),
      rope(getClockStr()), content, nil, nil, nil])
  # no analytics because context is not available

  if not writeRope(code, filename):
    rawMessage(conf, errCannotOpenFile, filename.string)
+
--- a/compiler/main.nim
+++ b/compiler/main.nim
@ -66,16 +66,8 @@ when not defined(leanCompiler):
    compileProject(graph)
    finishDoc2Pass(graph.config.projectName)

-proc setOutDir(conf: ConfigRef) =
-  if conf.outDir.isEmpty:
-    if optUseNimcache in conf.globalOptions:
-      conf.outDir = getNimcacheDir(conf)
-    else:
-      conf.outDir = conf.projectPath
-
 proc commandCompileToC(graph: ModuleGraph) =
  let conf = graph.config
-  setOutDir(conf)
  if conf.outFile.isEmpty:
    let base = conf.projectName
    let targetName = if optGenDynLib in conf.globalOptions:
@ -121,7 +113,6 @@ proc commandCompileToJS(graph: ModuleGraph) =
    let conf = graph.config
    conf.exc = excCpp

-    setOutDir(conf)
    if conf.outFile.isEmpty:
      conf.outFile = RelativeFile(conf.projectName & ".js")

@ -191,11 +182,6 @@ proc mainCommand*(graph: ModuleGraph) =
  conf.searchPaths.add(conf.libpath)
  setId(100)

-  ## Calling `setOutDir(conf)` unconditionally would fix regression
-  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
-  when false: setOutDir(conf)
-  if optUseNimcache in conf.globalOptions: setOutDir(conf)
-
  proc customizeForBackend(backend: TBackend) =
    ## Sets backend specific options but don't compile to backend yet in
    ## case command doesn't require it. This must be called by all commands.
@ -234,15 +220,40 @@ proc mainCommand*(graph: ModuleGraph) =
    of backendJs: commandCompileToJS(graph)
    of backendInvalid: doAssert false

+  template docLikeCmd(body) =
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+        else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
  ## process all backend commands
  case conf.command.normalize
  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
  of "cpp", "compiletocpp": compileToBackend(backendCpp)
  of "objc", "compiletooc": compileToBackend(backendObjc)
  of "js", "compiletojs": compileToBackend(backendJs)
-  of "r": # different from `"run"`!
-    conf.globalOptions.incl {optRun, optUseNimcache}
-    compileToBackend(backendC)
+  of "r": compileToBackend(backendC) # different from `"run"`!
  of "run":
    when hasTinyCBackend:
      extccomp.setCC(conf, "tcc", unknownLineInfo)
@ -254,29 +265,19 @@ proc mainCommand*(graph: ModuleGraph) =
  else: customizeForBackend(backendC) # fallback for other commands

  ## process all other commands
-  case conf.command.normalize
-  of "doc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
  of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
-      # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, eg for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
      commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
  of "rst2html":
    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
    when defined(leanCompiler):
@ -292,41 +293,10 @@ proc mainCommand*(graph: ModuleGraph) =
      conf.cmd = cmdRst2tex
      loadConfigs(DocTexConfig, cache, conf)
      commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
  of "gendepend":
    conf.cmd = cmdGenDepend
    commandGenDepend(graph)
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@ -510,7 +510,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
        if conf.hasHint(hintSource) and info != unknownLineInfo:
          conf.writeSurroundingSrc(info)
-        if conf.hasHint(hintMsgOrigin):
+        if hintMsgOrigin in conf.mainPackageNotes:
          styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
            " compiler msg initiated here", KindColor,
            KindFormat % hintMsgOrigin.msgToStr,
@ -529,6 +529,14 @@ template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
  conf.m.errorOutputs = {eStdOut, eStdErr}
  liMessage(conf, info, msg, arg, doAbort, instLoc())

+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    const info2 = instantiationInfo(-1, fullPaths = true)
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, info2)
+
 template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
  liMessage(conf, info, msg, arg, doRaise, instLoc())

--- a/compiler/nimpaths.nim
+++ b/compiler/nimpaths.nim
@ -0,0 +1,44 @@
+##[
+Represents absolute paths, but using a symbolic variables (eg $nimr) which can be
+resolved at runtime; this avoids hardcoding at compile time absolute paths so
+that the project root can be relocated.
+
+xxx consider some refactoring with $nim/testament/lib/stdtest/specialpaths.nim;
+specialpaths is simpler because it doesn't need variables to be relocatable at
+runtime (eg for use in testament)
+
+interpolation variables:
+  $nimr: such that `$nimr/lib/system.nim` exists (avoids confusion with $nim binary)
+         in compiler, it's obtainable via getPrefixDir(); for other tools (eg koch),
+        this could be getCurrentDir() or getAppFilename().parentDir.parentDir,
+        depending on use case
+
+Unstable API
+]##
+
+import std/[os,strutils]
+
+const
+  docCss* = "$nimr/doc/nimdoc.css"
+  docHackNim* = "$nimr/tools/dochack/dochack.nim"
+  docHackJs* = docHackNim.changeFileExt("js")
+  docHackJsFname* = docHackJs.lastPathPart
+  theindexFname* = "theindex.html"
+  nimdocOutCss* = "nimdoc.out.css"
+    # `out` to make it easier to use with gitignore in user's repos
+  htmldocsDirname* = "htmldocs"
+
+proc interp*(path: string, nimr: string): string =
+  result = path % ["nimr", nimr]
+  doAssert '$' notin result, $(path, nimr, result) # avoids un-interpolated variables in output
+
+proc getDocHacksJs*(nimr: string, nim = getCurrentCompilerExe(), forceRebuild = false): string =
+  ## return absolute path to dochhack.js, rebuilding if it doesn't exist or if
+  ## `forceRebuild`.
+  let docHackJs2 = docHackJs.interp(nimr = nimr)
+  if forceRebuild or not docHackJs2.fileExists:
+    let cmd =  "$nim js $file" % ["nim", nim.quoteShell, "file", docHackNim.interp(nimr = nimr).quoteShell]
+    echo "getDocHacksJs: cmd: " & cmd
+    doAssert execShellCmd(cmd) == 0, $(cmd)
+  doAssert docHackJs2.fileExists
+  result = docHackJs2
--- a/compiler/options.nim
+++ b/compiler/options.nim
@ -9,7 +9,7 @@

 import
  os, strutils, strtabs, sets, lineinfos, platform,
-  prefixmatches, pathutils
+  prefixmatches, pathutils, nimpaths

 from terminal import isatty
 from times import utc, fromUnix, local, getTime, format, DateTime
@ -518,8 +518,9 @@ const
  DefaultConfigNims* = RelativeFile"config.nims"
  DocConfig* = RelativeFile"nimdoc.cfg"
  DocTexConfig* = RelativeFile"nimdoc.tex.cfg"
-
-const oKeepVariableNames* = true
+  htmldocsDir* = htmldocsDirname.RelativeDir
+  docRootDefault* = "@default" # using `@` instead of `$` to avoid shell quoting complications
+  oKeepVariableNames* = true

 proc mainCommandArg*(conf: ConfigRef): string =
  ## This is intended for commands like check or parse
@ -543,6 +544,7 @@ proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): Absolute
  # explains regression https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
  # Yet another reason why "" should not mean ".";  `""/something` should raise
  # instead of implying "" == "." as it's bug prone.
+  doAssert conf.outDir.string.len > 0
  result = conf.outDir / changeFileExt(filename, ext)

 proc absOutFile*(conf: ConfigRef): AbsoluteFile =