fix #6583, fix #14376, index+search now generated for all projects, many bug fixes with nim doc (#14324)

* refs #6583 fix nim doc output
* changelog
* change default for outDir when unspecified
* cleanups
* --project implies --index

.gitignore

@@ -92,7 +92,8 @@ megatest.nim
 # ignore debug dirs generated by dsymutil on OSX
 *.dSYM
 
-nimdoc.out.css
-
 # for `nim c -r nimdoc/tester` etc; this can be in multiple places
 htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css

changelog.md

@@ -94,7 +94,6 @@
 - The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
 - The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
 
-- `osproc.execCmdEx` now takes an optional `input` for stdin.
 - `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
   parameters.
 
@@ -188,6 +187,9 @@ proc mydiv(a, b): int {.raises: [].} =
   and avoids polluting both $pwd and $projectdir. It can be used with any command.
 - `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override how an example is compiled and run,
   for example to change backend or compile only.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified (with or without `--project`);
+  passing `--project` now automatically generates an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.
 
 ## Tool changes
 

compiler/commands.nim

@@ -440,7 +440,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
   of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
   of "backend", "b":
     let backend = parseEnum(arg.normalize, TBackend.default)
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
@@ -485,7 +485,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:

compiler/docgen.nim

@@ -17,11 +17,10 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables
+  pathutils, trees, tables, nimpaths
 
 const
   exportSection = skField
-  htmldocsDir = RelativeDir"htmldocs"
   docCmdSkip = "skip"
 
 type
@@ -51,7 +50,7 @@ type
     destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
     exampleGroups: OrderedTable[string, ExampleGroup]
-    wroteCss*: bool
+    wroteSupportFiles*: bool
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
@@ -72,7 +71,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
   proc nimbleDir(): AbsoluteDir =
     getNimbleFile(conf, file2).parentDir.AbsoluteDir
   case conf.docRoot:
-  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+  of docRootDefault:
     result = getRelativePathFromConfigPath(conf, file)
     let dir = nimbleDir()
     if not dir.isEmpty:
@@ -88,9 +87,12 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
     result = getRelativePathFromConfigPath(conf, file)
     if result.isEmpty: bail()
   elif conf.docRoot.len > 0:
-    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
+    # we're (currently) requiring `isAbsolute` to avoid confusion when passing
-    doAssert conf.docRoot.existsDir, conf.docRoot
+    # a relative path (would it be relative wrt $PWD or to projectfile)
-    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+    conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
+    conf.globalAssert conf.docRoot.existsDir, arg=conf.docRoot
+    # needed because `canonicalizePath` called on `file`
+    result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
   else:
     bail()
   if isAbsolute(result.string):
@@ -1125,11 +1127,8 @@ proc genSection(d: PDoc, kind: TSymKind) =
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
       ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])
 
-const nimdocOutCss = "nimdoc.out.css"
+proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): Rope =
-  # `out` to make it easier to use with gitignore in user's repos
+  rope($relativeTo(outDir / linkto, destFile.splitFile().dir, '/'))
-
-proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
-  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
 
 proc genOutFile(d: PDoc): Rope =
   var
@@ -1160,15 +1159,17 @@ proc genOutFile(d: PDoc): Rope =
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
   content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref"],
       [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile, theindexFname.RelativeFile)])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
         "content", "author", "version", "analytics", "deprecationMsg"],
-        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        [relLink(d.conf.outDir, d.destFile, nimdocOutCss.RelativeFile),
+        relLink(d.conf.outDir, d.destFile, docHackJsFname.RelativeFile),
+        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
         content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
   else:
     code = content
@@ -1203,11 +1204,13 @@ proc writeOutput*(d: PDoc, useWarning = false) =
     if not writeRope(content, outfile):
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
-    elif not d.wroteCss:
+    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
-      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
+      let nimr = $d.conf.getPrefixDir()
-      let cssDest = $dir / nimdocOutCss
+      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
-      copyFile(cssSource, cssDest)
+      if optGenIndex in d.conf.globalOptions:
-      d.wroteCss = true
+        let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
+        copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
+      d.wroteSupportFiles = true
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
@@ -1234,6 +1237,8 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
 proc handleDocOutputOptions*(conf: ConfigRef) =
   if optWholeProject in conf.globalOptions:
     # Backward compatibility with previous versions
+    # xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
+    # it'd write to `sub/bar.html/main.html`
     conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
@@ -1308,20 +1313,23 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
     if not writeRope(content, filename):
       rawMessage(conf, errCannotOpenFile, filename.string)
 
-proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
+proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
-  var content = mergeIndexes(conf.projectFull.string).rope
+  var content = mergeIndexes(dir).rope
 
-  var outFile = RelativeFile"theindex"
+  var outFile = outFile
-  if conf.outFile != RelativeFile"":
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
-    outFile = conf.outFile
   let filename = getOutFile(conf, outFile, HtmlExt)
 
   let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+      "nimdoccss", "dochackjs",
+      "title", "tableofcontents", "moduledesc", "date", "time",
       "content", "author", "version", "analytics"],
-      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      rope"Index", nil, nil, rope(getDateStr()),
       rope(getClockStr()), content, nil, nil, nil])
   # no analytics because context is not available
 
   if not writeRope(code, filename):
     rawMessage(conf, errCannotOpenFile, filename.string)
+

compiler/main.nim

@@ -66,16 +66,8 @@ when not defined(leanCompiler):
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
-proc setOutDir(conf: ConfigRef) =
-  if conf.outDir.isEmpty:
-    if optUseNimcache in conf.globalOptions:
-      conf.outDir = getNimcacheDir(conf)
-    else:
-      conf.outDir = conf.projectPath
-
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-  setOutDir(conf)
   if conf.outFile.isEmpty:
     let base = conf.projectName
     let targetName = if optGenDynLib in conf.globalOptions:
@@ -121,7 +113,6 @@ proc commandCompileToJS(graph: ModuleGraph) =
     let conf = graph.config
     conf.exc = excCpp
 
-    setOutDir(conf)
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -191,11 +182,6 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.searchPaths.add(conf.libpath)
   setId(100)
 
-  ## Calling `setOutDir(conf)` unconditionally would fix regression
-  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
-  when false: setOutDir(conf)
-  if optUseNimcache in conf.globalOptions: setOutDir(conf)
-
   proc customizeForBackend(backend: TBackend) =
     ## Sets backend specific options but don't compile to backend yet in
     ## case command doesn't require it. This must be called by all commands.
@@ -234,15 +220,40 @@ proc mainCommand*(graph: ModuleGraph) =
     of backendJs: commandCompileToJS(graph)
     of backendInvalid: doAssert false
 
+  template docLikeCmd(body) =
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+        else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
   ## process all backend commands
   case conf.command.normalize
   of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
   of "cpp", "compiletocpp": compileToBackend(backendCpp)
   of "objc", "compiletooc": compileToBackend(backendObjc)
   of "js", "compiletojs": compileToBackend(backendJs)
-  of "r": # different from `"run"`!
+  of "r": compileToBackend(backendC) # different from `"run"`!
-    conf.globalOptions.incl {optRun, optUseNimcache}
-    compileToBackend(backendC)
   of "run":
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
@@ -254,29 +265,19 @@ proc mainCommand*(graph: ModuleGraph) =
   else: customizeForBackend(backendC) # fallback for other commands
 
   ## process all other commands
-  case conf.command.normalize
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
-  of "doc0":
+  of "doc0": docLikeCmd commandDoc(cache, conf)
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
   of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+    docLikeCmd():
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # of labels links in doc comments, eg for random.rand:
+        # because currently generates lots of false positives due to conflation
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-    when defined(leanCompiler):
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
     conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
@@ -292,41 +293,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
-    when defined(leanCompiler):
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
-      quit "compiler wasn't built with documentation generator"
+  of "ctags": docLikeCmd commandTags(cache, conf)
-    else:
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)

compiler/msgs.nim

@@ -510,7 +510,7 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
         styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
         if conf.hasHint(hintSource) and info != unknownLineInfo:
           conf.writeSurroundingSrc(info)
-        if conf.hasHint(hintMsgOrigin):
+        if hintMsgOrigin in conf.mainPackageNotes:
           styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
             " compiler msg initiated here", KindColor,
             KindFormat % hintMsgOrigin.msgToStr,
@@ -529,6 +529,14 @@ template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
   conf.m.errorOutputs = {eStdOut, eStdErr}
   liMessage(conf, info, msg, arg, doAbort, instLoc())
 
+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    const info2 = instantiationInfo(-1, fullPaths = true)
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, info2)
+
 template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
   liMessage(conf, info, msg, arg, doRaise, instLoc())
 

compiler/nimpaths.nim

@@ -0,0 +1,44 @@
+##[
+Represents absolute paths, but using a symbolic variables (eg $nimr) which can be
+resolved at runtime; this avoids hardcoding at compile time absolute paths so
+that the project root can be relocated.
+
+xxx consider some refactoring with $nim/testament/lib/stdtest/specialpaths.nim;
+specialpaths is simpler because it doesn't need variables to be relocatable at
+runtime (eg for use in testament)
+
+interpolation variables:
+  $nimr: such that `$nimr/lib/system.nim` exists (avoids confusion with $nim binary)
+         in compiler, it's obtainable via getPrefixDir(); for other tools (eg koch),
+        this could be getCurrentDir() or getAppFilename().parentDir.parentDir,
+        depending on use case
+
+Unstable API
+]##
+
+import std/[os,strutils]
+
+const
+  docCss* = "$nimr/doc/nimdoc.css"
+  docHackNim* = "$nimr/tools/dochack/dochack.nim"
+  docHackJs* = docHackNim.changeFileExt("js")
+  docHackJsFname* = docHackJs.lastPathPart
+  theindexFname* = "theindex.html"
+  nimdocOutCss* = "nimdoc.out.css"
+    # `out` to make it easier to use with gitignore in user's repos
+  htmldocsDirname* = "htmldocs"
+
+proc interp*(path: string, nimr: string): string =
+  result = path % ["nimr", nimr]
+  doAssert '$' notin result, $(path, nimr, result) # avoids un-interpolated variables in output
+
+proc getDocHacksJs*(nimr: string, nim = getCurrentCompilerExe(), forceRebuild = false): string =
+  ## return absolute path to dochhack.js, rebuilding if it doesn't exist or if
+  ## `forceRebuild`.
+  let docHackJs2 = docHackJs.interp(nimr = nimr)
+  if forceRebuild or not docHackJs2.fileExists:
+    let cmd =  "$nim js $file" % ["nim", nim.quoteShell, "file", docHackNim.interp(nimr = nimr).quoteShell]
+    echo "getDocHacksJs: cmd: " & cmd
+    doAssert execShellCmd(cmd) == 0, $(cmd)
+  doAssert docHackJs2.fileExists
+  result = docHackJs2

compiler/options.nim

@@ -9,7 +9,7 @@
 
 import
   os, strutils, strtabs, sets, lineinfos, platform,
-  prefixmatches, pathutils
+  prefixmatches, pathutils, nimpaths
 
 from terminal import isatty
 from times import utc, fromUnix, local, getTime, format, DateTime
@@ -518,8 +518,9 @@ const
   DefaultConfigNims* = RelativeFile"config.nims"
   DocConfig* = RelativeFile"nimdoc.cfg"
   DocTexConfig* = RelativeFile"nimdoc.tex.cfg"
-
+  htmldocsDir* = htmldocsDirname.RelativeDir
-const oKeepVariableNames* = true
+  docRootDefault* = "@default" # using `@` instead of `$` to avoid shell quoting complications
+  oKeepVariableNames* = true
 
 proc mainCommandArg*(conf: ConfigRef): string =
   ## This is intended for commands like check or parse
@@ -543,6 +544,7 @@ proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): Absolute
   # explains regression https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
   # Yet another reason why "" should not mean ".";  `""/something` should raise
   # instead of implying "" == "." as it's bug prone.
+  doAssert conf.outDir.string.len > 0
   result = conf.outDir / changeFileExt(filename, ext)
 
 proc absOutFile*(conf: ConfigRef): AbsoluteFile =

config/nimdoc.cfg

@@ -100,7 +100,7 @@ doc.body_toc = """
         <a href="lib.html">Standard library</a>
       </li>
       <li>
-        <a href="theindex.html">Index</a>
+        <a href="$theindexhref">Index</a>
       </li>
     </ul>
   </div>
@@ -143,7 +143,7 @@ doc.body_toc_group = """
         <a href="lib.html">Standard library</a>
       </li>
       <li>
-        <a href="theindex.html">Index</a>
+        <a href="$theindexhref">Index</a>
       </li>
     </ul>
   </div>
@@ -183,6 +183,9 @@ doc.body_toc_group = """
   </div>
   <div id="global-links">
     <ul class="simple">
+    <li>
+      <a href="$theindexhref">Index</a>
+    </li>
     </ul>
   </div>
   <div id="searchInputDiv">
@@ -239,7 +242,7 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <title>$title</title>
 <link rel="stylesheet" type="text/css" href="$nimdoccss">
 
-<script type="text/javascript" src="dochack.js"></script>
+<script type="text/javascript" src="$dochackjs"></script>
 
 <script type="text/javascript">
 function main() {

doc/docgen.rst

@@ -30,8 +30,16 @@ Generate HTML documentation for a whole project:
 
 ::
   # delete any htmldocs/*.idx file before starting
-  nim doc --project --index:on --git.url:<url> --git.commit:<tag> <main_filename>.nim
+  nim doc --project --index:on --git.url:<url> --git.commit:<tag> --outdir:htmldocs <main_filename>.nim
-  nim buildIndex -o:htmldocs/theindex.html htmldocs
+  # this will generate html files, a theindex.html index, css and js under `htmldocs`
+  # See also `--docroot` to specify a relative root.
+  # to get search (dochacks.js) to work locally, you need a server otherwise
+  # CORS will prevent opening file:// urls; this works:
+  python3 -m http.server 7029 --directory htmldocs
+  # When --outdir is omitted it defaults to $projectPath/htmldocs,
+  or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
+  and likewise without `--project`.
+  Adding `-r` will open in a browser directly.
 
 
 Documentation Comments

nimdoc/test_out_index_dot_html/expected/index.html

@@ -83,6 +83,9 @@ function main() {
   </div>
   <div id="global-links">
     <ul class="simple">
+    <li>
+      <a href="theindex.html">Index</a>
+    </li>
     </ul>
   </div>
   <div id="searchInputDiv">

nimdoc/testproject/expected/subdir/subdir_b/utils.html

@@ -20,7 +20,7 @@
 <title>subdir/subdir_b/utils</title>
 <link rel="stylesheet" type="text/css" href="../../nimdoc.out.css">
 
-<script type="text/javascript" src="dochack.js"></script>
+<script type="text/javascript" src="../../dochack.js"></script>
 
 <script type="text/javascript">
 function main() {
@@ -83,6 +83,9 @@ function main() {
   </div>
   <div id="global-links">
     <ul class="simple">
+    <li>
+      <a href="../../theindex.html">Index</a>
+    </li>
     </ul>
   </div>
   <div id="searchInputDiv">

nimdoc/testproject/expected/testproject.html

@@ -83,6 +83,9 @@ function main() {
   </div>
   <div id="global-links">
     <ul class="simple">
+    <li>
+      <a href="theindex.html">Index</a>
+    </li>
     </ul>
   </div>
   <div id="searchInputDiv">

testament/lib/stdtest/specialpaths.nim

@@ -1,5 +1,6 @@
 #[
 todo: move findNimStdLibCompileTime, findNimStdLib here
+xxx: consider moving this to $nim/compiler/relpaths.nim to get relocatable paths
 ]#
 
 import os
@@ -9,22 +10,17 @@ import os
 # This means the binaries they produce will embed hardcoded paths, which
 # isn't appropriate for some applications that need to be relocatable.
 
-const sourcePath = currentSourcePath()
+const
-  # robust way to derive other paths here
+  sourcePath = currentSourcePath()
-  # We don't depend on PATH so this is robust to having multiple nim
+    # robust way to derive other paths here
-  # binaries
+    # We don't depend on PATH so this is robust to having multiple nim binaries
-
+  nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir ## root of Nim repo
-const nimRootDir* = sourcePath.parentDir.parentDir.parentDir.parentDir
+  stdlibDir* = nimRootDir / "lib"
-  ## root of Nim repo
+  systemPath* = stdlibDir / "system.nim"
-
+  testsDir* = nimRootDir / "tests"
-const stdlibDir* = nimRootDir / "lib"
+  buildDir* = nimRootDir / "build"
-  # todo: make nimeval.findNimStdLibCompileTime use this
+    ## refs #10268: all testament generated files should go here to avoid
-
+    ## polluting .gitignore
-const systemPath* = stdlibDir / "system.nim"
-
-const buildDir* = nimRootDir / "build"
-  ## refs #10268: all testament generated files should go here to avoid
-  ## polluting .gitignore
 
 static:
   # sanity check

testament/lib/stdtest/unittest_light.nim

@@ -1,5 +1,3 @@
-# note: consider merging tests/assert/testhelper.nim here.
-
 proc mismatch*[T](lhs: T, rhs: T): string =
   ## Simplified version of `unittest.require` that satisfies a common use case,
   ## while avoiding pulling too many dependencies. On failure, diagnostic
@@ -31,8 +29,8 @@ proc mismatch*[T](lhs: T, rhs: T): string =
       result.add "lhs[0..<i]:{" & replaceInvisible($lhs[
           0..<i]) & "}"
 
-proc assertEquals*[T](lhs: T, rhs: T) =
+proc assertEquals*[T](lhs: T, rhs: T, msg = "") =
   when false: # can be useful for debugging to see all that's fed to this.
     echo "----" & $lhs
   if lhs!=rhs:
-    doAssert false, mismatch(lhs, rhs)
+    doAssert false, mismatch(lhs, rhs) & "\n" & msg

tests/misc/trunner.nim

@@ -8,19 +8,20 @@ discard """
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
 
 import std/[strformat,os,osproc,unittest]
+from std/sequtils import toSeq,mapIt
+from std/algorithm import sorted
+import stdtest/[specialpaths, unittest_light]
 
-const nim = getCurrentCompilerExe()
+import "$nim/compiler/nimpaths"
 
-const mode =
+const
-  when defined(c): "c"
+  nim = getCurrentCompilerExe()
-  elif defined(cpp): "cpp"
+  mode =
-  else: static: doAssert false
+    when defined(c): "c"
-
+    elif defined(cpp): "cpp"
-const testsDir = currentSourcePath.parentDir.parentDir
+    else: static: doAssert false
-const buildDir = testsDir.parentDir / "build"
+  nimcache = buildDir / "nimcacheTrunner"
-const nimcache = buildDir / "nimcacheTrunner"
+    # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests
-  # `querySetting(nimcacheDir)` would also be possible, but we thus
-  # avoid stomping on other parallel tests
 
 proc runCmd(file, options = ""): auto =
   let fileabs = testsDir / file.unixToNativePath
@@ -63,6 +64,82 @@ else: # don't run twice the same test
   import std/[strutils]
   template check2(msg) = doAssert msg in output, output
 
+  block: # tests with various options `nim doc --project --index --docroot`
+    # regression tests for issues and PRS: #14376 #13223 #6583 ##13647
+    let file = testsDir / "nimdoc/sub/mmain.nim"
+    let mainFname = "mmain.html"
+    let htmldocsDirCustom = nimcache / "htmldocsCustom"
+    let docroot = testsDir / "nimdoc"
+    let options = [
+      0: "--project",
+      1: "--project --docroot",
+      2: "",
+      3: fmt"--outDir:{htmldocsDirCustom}",
+      4: fmt"--docroot:{docroot}",
+      5: "--project --useNimcache",
+      6: "--index:off",
+    ]
+
+    for i in 0..<options.len:
+      let htmldocsDir = case i
+      of 3: htmldocsDirCustom
+      of 5: nimcache / htmldocsDirname
+      else: file.parentDir / htmldocsDirname
+
+      var cmd = fmt"{nim} doc --index:on --listFullPaths --hint:successX:on --nimcache:{nimcache} {options[i]} {file}"
+      removeDir(htmldocsDir)
+      let (outp, exitCode) = execCmdEx(cmd)
+      check exitCode == 0
+      proc nativeToUnixPathWorkaround(a: string): string =
+        # xxx pending https://github.com/nim-lang/Nim/pull/13265 `nativeToUnixPath`
+        a.replace(DirSep, '/')
+
+      let ret = toSeq(walkDirRec(htmldocsDir, relative=true)).mapIt(it.nativeToUnixPathWorkaround).sorted.join("\n")
+      let context = $(i, ret, cmd)
+      var expected = ""
+      case i
+      of 0,5:
+        let htmlFile = htmldocsDir/"mmain.html"
+        check htmlFile in outp # sanity check for `hintSuccessX`
+        assertEquals ret, """
+@@/imp.html
+@@/imp.idx
+dochack.js
+imp.html
+imp.idx
+imp2.html
+imp2.idx
+mmain.html
+mmain.idx
+nimdoc.out.css
+theindex.html""", context
+      of 1: assertEquals ret, """
+dochack.js
+nimdoc.out.css
+tests/nimdoc/imp.html
+tests/nimdoc/imp.idx
+tests/nimdoc/sub/imp.html
+tests/nimdoc/sub/imp.idx
+tests/nimdoc/sub/imp2.html
+tests/nimdoc/sub/imp2.idx
+tests/nimdoc/sub/mmain.html
+tests/nimdoc/sub/mmain.idx
+theindex.html"""
+      of 2, 3: assertEquals ret, """
+dochack.js
+mmain.html
+mmain.idx
+nimdoc.out.css""", context
+      of 4: assertEquals ret, """
+dochack.js
+nimdoc.out.css
+sub/mmain.html
+sub/mmain.idx""", context
+      of 6: assertEquals ret, """
+mmain.html
+nimdoc.out.css""", context
+      else: doAssert false
+
   block: # mstatic_assert
     let (output, exitCode) = runCmd("ccgbugs/mstatic_assert.nim", "-d:caseBad")
     check2 "sizeof(bool) == 2"
@@ -121,7 +198,7 @@ else: # don't run twice the same test
       let cmd = fmt"""{nim} doc -b:{backend} --nimcache:{nimcache} -d:m13129Foo1 "--doccmd:-d:m13129Foo2 --hints:off" --usenimcache --hints:off {file}"""
       check execCmdEx(cmd) == (&"ok1:{backend}\nok2: backend: {backend}\n", 0)
     # checks that --usenimcache works with `nim doc`
-    check fileExists(nimcache / "m13129.html")
+    check fileExists(nimcache / "htmldocs/m13129.html")
 
     block: # mak sure --backend works with `nim r`
       let cmd = fmt"{nim} r --backend:{mode} --hints:off --nimcache:{nimcache} {file}"

tests/nimdoc/imp.nim

@@ -0,0 +1 @@
+proc fn5*() = discard

tests/nimdoc/m13129.nim

@@ -17,11 +17,9 @@ proc main*() =
     doAssert defined(m13129Foo2)
     doAssert not defined(nimdoc)
     echo "ok2: backend: " & querySetting(backend)
-    # echo defined(c), defined(js), 
 
 import std/compilesettings
 when defined nimdoc:
-  # import std/compilesettings
   static:
     doAssert defined(m13129Foo1)
     doAssert not defined(m13129Foo2)
@@ -33,6 +31,6 @@ when isMainModule:
     let cache = querySetting(nimcacheDir)
     doAssert cache.len > 0
     let app = getAppFilename()
-    doAssert app.isRelativeTo(cache)
+    doAssert app.isRelativeTo(cache), $(app, cache)
     doAssert querySetting(projectFull) == currentSourcePath
     echo "ok3"

tests/nimdoc/sub/imp.nim

@@ -0,0 +1 @@
+proc fn4*() = discard

tests/nimdoc/sub/imp2.nim

@@ -0,0 +1 @@
+proc fn3*() = discard

tests/nimdoc/sub/mmain.nim

@@ -0,0 +1,8 @@
+{.warning[UnusedImport]: off.}
+
+import ../imp as impa
+import imp as impb
+import imp2
+
+proc fn1*() = discard
+proc fn2*() = discard

tools/kochdocs.nim

@@ -1,6 +1,7 @@
 ## Part of 'koch' responsible for the documentation generation.
 
 import os, strutils, osproc, sets, pathnorm
+import "../compiler/nimpaths"
 
 const
   gaCode* = " --doc.googleAnalytics:UA-48159761-1"
@@ -9,7 +10,6 @@ const
   gitUrl = "https://github.com/nim-lang/Nim"
   docHtmlOutput = "doc/html"
   webUploadOutput = "web/upload"
-  docHackDir = "tools/dochack"
 
 var nimExe*: string
 
@@ -289,20 +289,18 @@ proc buildPdfDoc*(nimArgs, destPath: string) =
       removeFile(changeFileExt(pdf, "out"))
       removeFile(changeFileExt(d, "tex"))
 
-proc buildJS() =
+proc buildJS(): string =
-  exec(findNim().quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" %
+  let nim = findNim()
+  exec(nim.quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" %
       [webUploadOutput / "nimblepkglist.js"])
-  exec(findNim().quoteShell() & " js " & (docHackDir / "dochack.nim"))
+  result = getDocHacksJs(nimr = getCurrentDir(), nim)
 
 proc buildDocs*(args: string) =
-  const
-    docHackJs = "dochack.js"
   let
     a = nimArgs & " " & args
-    docHackJsSource = docHackDir / docHackJs
+    docHackJsSource = buildJS()
-    docHackJsDest = docHtmlOutput / docHackJs
+    docHackJs = docHackJsSource.lastPathPart
 
-  buildJS()                     # This call generates docHackJsSource
   let docup = webUploadOutput / NimVersion
   createDir(docup)
   buildDocSamples(a, docup)
@@ -313,5 +311,5 @@ proc buildDocs*(args: string) =
   createDir(docHtmlOutput)
   buildDocSamples(nimArgs, docHtmlOutput)
   buildDoc(nimArgs, docHtmlOutput)
-  copyFile(docHackJsSource, docHackJsDest)
+  copyFile(docHackJsSource, docHtmlOutput / docHackJs)
   copyFile(docHackJsSource, docup / docHackJs)