fix #13150 nim doc --project now works reliably (#13223)

* fix #13150 `nim doc --project` works with duplicate names and with imports below main project file

* add to help; fixup after #13212 isRelativeTo got merged
* fix test tests/compilerapi/tcompilerapi.nim
* remove nimblePkg field; compute on the fly instead
* kochdocs: compiler docs now under compiler/
* --docRoot now has smart default: best among @pkg, @path

compiler/ast.nim

@@ -1026,6 +1026,26 @@ const
   defaultAlignment = -1
   defaultOffset = -1
 
+
+proc getnimblePkg*(a: PSym): PSym =
+  result = a
+  while result != nil:
+    case result.kind
+    of skModule:
+      result = result.owner
+      assert result.kind == skPackage
+    of skPackage:
+      if result.owner == nil:
+        break
+      else:
+        result = result.owner
+    else:
+      assert false, $result.kind
+
+proc getnimblePkgId*(a: PSym): int =
+  let b = a.getnimblePkg
+  result = if b == nil: -1 else: b.id
+
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
 #var
 #  gMainPackageId*: int

compiler/commands.nim

@@ -413,6 +413,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
+  of "docroot":
+    conf.docRoot = if arg.len == 0: "@default" else: arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":

compiler/docgen.nim

@@ -48,6 +48,47 @@ type
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
+proc nativeToUnix(path: string): string =
+  doAssert not path.isAbsolute # absolute files need more care for the drive
+  when DirSep == '\\':
+    result = replace(path, '\\', '/')
+  else: result = path
+
+proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile =
+  ## returns a relative file that will be appended to outDir
+  let file2 = $file
+  template bail() =
+    result = relativeTo(file, conf.projectPath)
+  proc nimbleDir(): AbsoluteDir =
+    getNimbleFile(conf, file2).parentDir.AbsoluteDir
+  case conf.docRoot:
+  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+    result = getRelativePathFromConfigPath(conf, file)
+    let dir = nimbleDir()
+    if not dir.isEmpty:
+      let result2 = relativeTo(file, dir)
+      if not result2.isEmpty and (result.isEmpty or result2.string.len < result.string.len):
+        result = result2
+    if result.isEmpty: bail()
+  of "@pkg":
+    let dir = nimbleDir()
+    if dir.isEmpty: bail()
+    else: result = relativeTo(file, dir)
+  of "@path":
+    result = getRelativePathFromConfigPath(conf, file)
+    if result.isEmpty: bail()
+  elif conf.docRoot.len > 0:
+    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
+    doAssert conf.docRoot.existsDir, conf.docRoot
+    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+  else:
+    bail()
+  if isTitle:
+    result = result.string.nativeToUnix.RelativeFile
+  else:
+    result = result.string.replace("..", "@@").RelativeFile ## refs #13223
+  doAssert not result.isEmpty
+
 proc whichType(d: PDoc; n: PNode): PSym =
   if n.kind == nkSym:
     if d.types.strTableContains(n.sym):
@@ -175,7 +216,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       if execShellCmd(c2) != status:
         rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
   result.emitted = initIntSet()
-  result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath),
+  result.destFile = getOutFile2(conf, presentationPath(conf, filename),
                                 outExt, htmldocsDir, false)
   result.thisDir = result.destFile.splitFile.dir
 
@@ -295,13 +336,12 @@ proc getPlainDocstring(n: PNode): string =
       if result.len > 0: return
 
 proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
-  result = module.kind == skModule and module.owner != nil and
+  result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId
-      module.owner.id == conf.mainPackageId
 
 proc externalDep(d: PDoc; module: PSym): string =
-  if optWholeProject in d.conf.globalOptions:
+  if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
     let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
-    let tmp = getOutFile2(d.conf, full.relativeTo(d.conf.projectPath), HtmlExt,
+    let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt,
                           htmldocsDir, sfMainModule notin module.flags)
     result = relativeTo(tmp, d.thisDir, '/').string
   else:
@@ -1031,11 +1071,12 @@ proc genOutFile(d: PDoc): Rope =
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
     title = d.meta[metaTitle]
-    let external = AbsoluteFile(d.filename).relativeTo(d.conf.projectPath, '/').changeFileExt(HtmlExt).string
+    let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnix
     setIndexTerm(d[], external, "", title)
   else:
     # Modules get an automatic title for the HTML, but no entry in the index.
-    title = extractFilename(changeFileExt(d.filename, ""))
+    # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups
+    title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("")
 
   let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
@@ -1060,10 +1101,14 @@ proc generateIndex*(d: PDoc) =
     let dir = if not d.conf.outDir.isEmpty: d.conf.outDir
               else: d.conf.projectPath / htmldocsDir
     createDir(dir)
-    let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename,
+    let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
-                                              d.conf.projectPath), IndexExt)
     writeIndexFile(d[], dest.string)
 
+proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
+  if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html
+    if d.conf.outFile.isEmpty and not d.conf.outDir.isEmpty:
+      d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+
 proc writeOutput*(d: PDoc, useWarning = false) =
   runAllExamples(d)
   var content = genOutFile(d)
@@ -1073,7 +1118,7 @@ proc writeOutput*(d: PDoc, useWarning = false) =
     template outfile: untyped = d.destFile
     #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt, htmldocsDir)
     createDir(outfile.splitFile.dir)
-    d.conf.outFile = outfile.extractFilename.RelativeFile
+    updateOutfile(d, outfile)
     if not writeRope(content, outfile):
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
@@ -1100,7 +1145,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
     if open(f, d.destFile.string, fmWrite):
       write(f, $content)
       close(f)
-      d.conf.outFile = d.destFile.extractFilename.RelativeFile
+      updateOutfile(d, d.destFile)
     else:
       localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1),
                  warnUser, "unable to open file \"" & d.destFile.string &

compiler/docgen2.nim

@@ -22,8 +22,8 @@ type
     config: ConfigRef
   PGen = ref TGen
 
-template shouldProcess(g): bool =
+proc shouldProcess(g: PGen): bool =
-  (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =

compiler/main.nim

@@ -293,7 +293,9 @@ proc mainCommand*(graph: ModuleGraph) =
       for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
+      var lazyPaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
+      for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string)
 
       var hints = newJObject() # consider factoring with `listHints`
       for a in hintMin..hintMax:
@@ -311,6 +313,7 @@ proc mainCommand*(graph: ModuleGraph) =
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
         (key: "lib_paths", val: %libpaths),
+        (key: "lazyPaths", val: %lazyPaths),
         (key: "outdir", val: %conf.outDir.string),
         (key: "out", val: %conf.outFile.string),
         (key: "nimcache", val: %getNimcacheDir(conf).string),

compiler/modules.nim

@@ -39,7 +39,8 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
         # but starting with version 0.20 we now produce a fake Nimble package instead
         # to resolve the conflicts:
         let pck3 = fakePackageName(graph.config, filename)
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
+        # this makes the new `packSym`'s owner be the original `packSym`
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, result.info)
         initStrTable(packSym.tab)
         graph.packageSyms.strTableAdd(packSym)
 

compiler/options.nim

@@ -263,6 +263,7 @@ type
     implicitIncludes*: seq[string] # modules that are to be implicitly included
     docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \
     # The string uses the formatting variables `path` and `line`.
+    docRoot*: string ## see nim --fullhelp for --docRoot
 
      # the used compiler
     cIncludes*: seq[AbsoluteDir]  # directories to search for included files
@@ -656,6 +657,24 @@ template patchModule(conf: ConfigRef) {.dirty.} =
       let ov = conf.moduleOverrides[key]
       if ov.len > 0: result = AbsoluteFile(ov)
 
+when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
+  proc isRelativeTo(path, base: string): bool =
+    # pending #13212 use os.isRelativeTo
+    let path = path.normalizedPath
+    let base = base.normalizedPath
+    let ret = relativePath(path, base)
+    result = path.len > 0 and not ret.startsWith ".."
+
+proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeFile =
+  let f = $f
+  template search(paths) =
+    for it in paths:
+      let it = $it
+      if f.isRelativeTo(it):
+        return relativePath(f, it).RelativeFile
+  search(conf.searchPaths)
+  search(conf.lazyPaths)
+
 proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile =
   if f.isAbsolute:
     result = if f.existsFile: AbsoluteFile(f) else: AbsoluteFile""

compiler/packagehandling.nim

@@ -15,10 +15,8 @@ iterator myParentDirs(p: string): string =
     if current.len == 0: break
     yield current
 
-proc resetPackageCache*(conf: ConfigRef) =
+proc getNimbleFile*(conf: ConfigRef; path: string): string =
-  conf.packageCache = newPackageCache()
+  ## returns absolute path to nimble file, eg: /pathto/cligen.nimble
-
-proc getPackageName*(conf: ConfigRef; path: string): string =
   var parents = 0
   block packageSearch:
     for d in myParentDirs(path):
@@ -27,7 +25,7 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
         return conf.packageCache[d]
       inc parents
       for file in walkFiles(d / "*.nimble"):
-        result = file.splitFile.name
+        result = file
         break packageSearch
   # we also store if we didn't find anything:
   when not defined(nimNoNilSeqs):
@@ -38,6 +36,11 @@ proc getPackageName*(conf: ConfigRef; path: string): string =
     dec parents
     if parents <= 0: break
 
+proc getPackageName*(conf: ConfigRef; path: string): string =
+  ## returns nimble package name, eg: `cligen`
+  let path = getNimbleFile(conf, path)
+  result = path.splitFile.name
+
 proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
   # Convert `path` so that 2 modules with same name
   # in different directory get different name and they can be

compiler/pathutils.nim

@@ -67,7 +67,7 @@ when true:
 
   proc `/`*(base: AbsoluteDir; f: RelativeFile): AbsoluteFile =
     let base = postProcessBase(base)
-    assert(not isAbsolute(f.string))
+    assert(not isAbsolute(f.string), f.string)
     result = AbsoluteFile newStringOfCap(base.string.len + f.string.len)
     var state = 0
     addNormalizePath(base.string, result.string, state)
@@ -83,7 +83,10 @@ when true:
 
   proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir;
                    sep = DirSep): RelativeFile =
-    RelativeFile(relativePath(fullPath.string, baseFilename.string, sep))
+    # this currently fails for `tests/compilerapi/tcompilerapi.nim`
+    # it's needed otherwise would returns an absolute path
+    # assert not baseFilename.isEmpty, $fullPath
+    result = RelativeFile(relativePath(fullPath.string, baseFilename.string, sep))
 
   proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile =
     if isAbsolute(file): result = AbsoluteFile(file)

doc/advopt.txt

@@ -67,6 +67,13 @@ Advanced options:
   --clib:LIBNAME            link an additional C library
                             (you should omit platform-specific extensions)
   --project                 document the whole project (doc2)
+  --docRoot:path            nim doc --docRoot:/foo --project --outdir:docs /foo/sub/main.nim
+                            generates: docs/sub/main.html
+                            if path == @pkg, will use nimble file enclosing dir
+                            if path == @path, will use first matching dir in --path
+                            if path == @default (the default and most useful), will use
+                            best match among @pkg,@path.
+                            if these are nonexistant, will use project path
   --docSeeSrcUrl:url        activate 'see source' for doc and doc2 commands
                             (see doc.item.seesrc in config/nimdoc.cfg)
   --docInternal             also generate documentation for non-exported symbols

nimdoc/testproject/expected/subdir/subdir_b/utils.html

@@ -17,7 +17,7 @@
 <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 
 <!-- CSS -->
-<title>utils</title>
+<title>subdir/subdir_b/utils</title>
 <link rel="stylesheet" type="text/css" href="../../nimdoc.out.css">
 
 <script type="text/javascript" src="dochack.js"></script>
@@ -71,7 +71,7 @@ function main() {
 <body onload="main()">
 <div class="document" id="documentId">
   <div class="container">
-    <h1 class="title">utils</h1>
+    <h1 class="title">subdir/subdir_b/utils</h1>
     <div class="row">
   <div class="three columns">
   <div class="theme-switch-wrapper">

tools/kochdocs.nim

@@ -319,9 +319,11 @@ proc buildDoc(nimArgs, destPath: string) =
       destPath / changeFileExt(splitFile(d).name, "html"), d]
     i.inc
   for d in items(doc):
-    commands[i] = nim & " doc $# --git.url:$# -o:$# --index:on $#" %
+    var nimArgs2 = nimArgs
-      [nimArgs, gitUrl,
+    if d.isRelativeTo("compiler"):
-      destPath / changeFileExt(splitFile(d).name, "html"), d]
+      nimArgs2.add " --docroot"
+    commands[i] = nim & " doc $# --git.url:$# --outdir:$# --index:on $#" %
+      [nimArgs2, gitUrl, destPath, d]
     i.inc
   for d in items(withoutIndex):
     commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" %