fix #6583, fix #14376, index+search now generated for all projects, many bug fixes with nim doc (#14324)

* refs #6583 fix nim doc output
* changelog
* change default for outDir when unspecified
* cleanups
* --project implies --index

compiler/main.nim

@@ -66,16 +66,8 @@ when not defined(leanCompiler):
     compileProject(graph)
     finishDoc2Pass(graph.config.projectName)
 
-proc setOutDir(conf: ConfigRef) =
-  if conf.outDir.isEmpty:
-    if optUseNimcache in conf.globalOptions:
-      conf.outDir = getNimcacheDir(conf)
-    else:
-      conf.outDir = conf.projectPath
-
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-  setOutDir(conf)
   if conf.outFile.isEmpty:
     let base = conf.projectName
     let targetName = if optGenDynLib in conf.globalOptions:
@@ -121,7 +113,6 @@ proc commandCompileToJS(graph: ModuleGraph) =
     let conf = graph.config
     conf.exc = excCpp
 
-    setOutDir(conf)
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -191,11 +182,6 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.searchPaths.add(conf.libpath)
   setId(100)
 
-  ## Calling `setOutDir(conf)` unconditionally would fix regression
-  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
-  when false: setOutDir(conf)
-  if optUseNimcache in conf.globalOptions: setOutDir(conf)
-
   proc customizeForBackend(backend: TBackend) =
     ## Sets backend specific options but don't compile to backend yet in
     ## case command doesn't require it. This must be called by all commands.
@@ -234,15 +220,40 @@ proc mainCommand*(graph: ModuleGraph) =
     of backendJs: commandCompileToJS(graph)
     of backendInvalid: doAssert false
 
+  template docLikeCmd(body) =
+    when defined(leanCompiler):
+      quit "compiler wasn't built with documentation generator"
+    else:
+      wantMainModule(conf)
+      conf.cmd = cmdDoc
+      loadConfigs(DocConfig, cache, conf)
+      defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+        else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
   ## process all backend commands
   case conf.command.normalize
   of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
   of "cpp", "compiletocpp": compileToBackend(backendCpp)
   of "objc", "compiletooc": compileToBackend(backendObjc)
   of "js", "compiletojs": compileToBackend(backendJs)
-  of "r": # different from `"run"`!
-    conf.globalOptions.incl {optRun, optUseNimcache}
-    compileToBackend(backendC)
+  of "r": compileToBackend(backendC) # different from `"run"`!
   of "run":
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
@@ -254,29 +265,19 @@ proc mainCommand*(graph: ModuleGraph) =
   else: customizeForBackend(backendC) # fallback for other commands
 
   ## process all other commands
-  case conf.command.normalize
-  of "doc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
   of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
-      # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, eg for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
     conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
@@ -292,41 +293,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)