fixes #4485; package handling works better; docgen works with --project on Nimble package level

compiler/ast.nim

@@ -771,7 +771,7 @@ type
       procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
       #scope*: PScope          # the scope where the proc was defined
-    of skModule:
+    of skModule, skPackage:
       # modules keep track of the generic symbols they use from other modules.
       # this is because in incremental compilation, when a module is about to
       # be replaced with a newer version, we must decrement the usage count

compiler/docgen.nim

@@ -650,22 +650,32 @@ proc generateIndex*(d: PDoc) =
     writeIndexFile(d[], splitFile(options.outFile).dir /
                         splitFile(d.filename).name & IndexExt)
 
+proc getOutFile2(filename, ext, dir: string): string =
+  if gWholeProject:
+    let d = if options.outFile != "": options.outFile else: dir
+    createDir(d)
+    result = d / changeFileExt(filename, ext)
+  else:
+    result = getOutFile(filename, ext)
+
 proc writeOutput*(d: PDoc, filename, outExt: string, useWarning = false) =
   var content = genOutFile(d)
   if optStdout in gGlobalOptions:
     writeRope(stdout, content)
   else:
-    writeRope(content, getOutFile(filename, outExt), useWarning)
+    writeRope(content, getOutFile2(filename, outExt, "htmldoc"), useWarning)
 
 proc writeOutputJson*(d: PDoc, filename, outExt: string,
                       useWarning = false) =
-  let content = $d.jArray
+  let content = %*{"orig": d.filename,
+    "nimble": getPackageName(d.filename),
+    "entries": d.jArray}
   if optStdout in gGlobalOptions:
-    write(stdout, content)
+    write(stdout, $content)
   else:
     var f: File
-    if open(f, getOutFile(filename, outExt), fmWrite):
+    if open(f, getOutFile2(filename, outExt, "jsondoc"), fmWrite):
-      write(f, content)
+      write(f, $content)
       close(f)
     else:
       discard "fixme: error report"

compiler/docgen2.nim

@@ -19,25 +19,25 @@ type
     module: PSym
   PGen = ref TGen
 
-proc close(p: PPassContext, n: PNode): PNode =
+template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
-  if gWholeProject or sfMainModule in g.module.flags:
+  echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId
-    writeOutput(g.doc, g.module.filename, HtmlExt, useWarning)
+  if (g.module.owner.id == gMainPackageId and gWholeProject) or
+    sfMainModule in g.module.flags:
+    body
     try:
       generateIndex(g.doc)
     except IOError:
       discard
 
+proc close(p: PPassContext, n: PNode): PNode =
+  closeImpl:
+    writeOutput(g.doc, g.module.filename, HtmlExt, useWarning)
+
 proc closeJson(p: PPassContext, n: PNode): PNode =
-  var g = PGen(p)
+  closeImpl:
-  let useWarning = sfMainModule notin g.module.flags
-  if gWholeProject or sfMainModule in g.module.flags:
     writeOutputJson(g.doc, g.module.filename, ".json", useWarning)
-    try:
-      generateIndex(g.doc)
-    except IOError:
-      discard
 
 proc processNode(c: PPassContext, n: PNode): PNode =
   result = n

compiler/modules.nim

@@ -30,6 +30,9 @@ var
     ## XXX: we should implement recycling of file IDs
     ## if the user keeps renaming modules, the file IDs will keep growing
   gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
+  packageSyms: TStrTable
+
+initStrTable(packageSyms)
 
 proc getModule*(fileIdx: int32): PSym =
   if fileIdx >= 0 and fileIdx < gCompiledModules.len:
@@ -91,6 +94,7 @@ proc resetAllModules* =
     if gCompiledModules[i] != nil:
       resetModule(i.int32)
   resetPackageCache()
+  initStrTable(packageSyms)
   # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
 proc resetAllModulesHard* =
@@ -98,6 +102,7 @@ proc resetAllModulesHard* =
   gCompiledModules.setLen 0
   gMemCacheData.setLen 0
   magicsys.resetSysTypes()
+  initStrTable(packageSyms)
   # XXX
   #gOwners = @[]
 
@@ -140,8 +145,16 @@ proc newModule(fileIdx: int32): PSym =
     rawMessage(errInvalidModuleName, result.name.s)
 
   result.info = newLineInfo(fileIdx, 1, 1)
-  result.owner = newSym(skPackage, getIdent(getPackageName(filename)), nil,
+  let pack = getIdent(getPackageName(filename))
-                        result.info)
+  var packSym = packageSyms.strTableGet(pack)
+  if packSym == nil:
+    let pck = getPackageName(filename)
+    let pck2 = if pck.len > 0: pck else: "unknown"
+    packSym = newSym(skPackage, getIdent(pck2), nil, result.info)
+    initStrTable(packSym.tab)
+    packageSyms.strTableAdd(packSym)
+
+  result.owner = packSym
   result.position = fileIdx
 
   growCache gMemCacheData, fileIdx
@@ -151,6 +164,11 @@ proc newModule(fileIdx: int32): PSym =
   incl(result.flags, sfUsed)
   initStrTable(result.tab)
   strTableAdd(result.tab, result) # a module knows itself
+  let existing = strTableGet(packSym.tab, result.name)
+  if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+    localError(result.info, "module names need to be unique per Nimble package; module clashes with " & existing.info.fileIndex.toFullPath)
+  # strTableIncl() for error corrections:
+  discard strTableIncl(packSym.tab, result)
 
 proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym =
   result = getModule(fileIdx)

compiler/options.nim

@@ -239,6 +239,8 @@ proc removeTrailingDirSep*(path: string): string =
   else:
     result = path
 
+include packagehandling
+
 proc getNimcacheDir*: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir /
                                                          genSubDir
@@ -258,54 +260,6 @@ proc pathSubs*(p, config: string): string =
   if '~' in result:
     result = result.replace("~", home)
 
-template newPackageCache(): expr =
-  newStringTable(when FileSystemCaseSensitive:
-                   modeCaseInsensitive
-                 else:
-                   modeCaseSensitive)
-
-var packageCache = newPackageCache()
-
-proc resetPackageCache*() = packageCache = newPackageCache()
-
-iterator myParentDirs(p: string): string =
-  # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug...
-  var current = p
-  while true:
-    current = current.parentDir
-    if current.len == 0: break
-    yield current
-
-proc getPackageName*(path: string): string =
-  var parents = 0
-  block packageSearch:
-    for d in myParentDirs(path):
-      if packageCache.hasKey(d):
-        #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name
-        return packageCache[d]
-      inc parents
-      for file in walkFiles(d / "*.nimble"):
-        result = file.splitFile.name
-        break packageSearch
-      for file in walkFiles(d / "*.babel"):
-        result = file.splitFile.name
-        break packageSearch
-  # we also store if we didn't find anything:
-  if result.isNil: result = ""
-  for d in myParentDirs(path):
-    #echo "set cache ", d, " |", result, "|", parents
-    packageCache[d] = result
-    dec parents
-    if parents <= 0: break
-
-proc withPackageName*(path: string): string =
-  let x = path.getPackageName
-  if x.len == 0:
-    result = path
-  else:
-    let (p, file, ext) = path.splitFile
-    result = (p / (x & '_' & file)) & ext
-
 proc toGeneratedFile*(path, ext: string): string =
   ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
   var (head, tail) = splitPath(path)

compiler/packagehandling.nim

@@ -0,0 +1,56 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+iterator myParentDirs(p: string): string =
+  # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug...
+  var current = p
+  while true:
+    current = current.parentDir
+    if current.len == 0: break
+    yield current
+
+template newPackageCache(): expr =
+  newStringTable(when FileSystemCaseSensitive:
+                   modeCaseInsensitive
+                 else:
+                   modeCaseSensitive)
+
+var packageCache = newPackageCache()
+
+proc resetPackageCache*() = packageCache = newPackageCache()
+
+proc getPackageName*(path: string): string =
+  var parents = 0
+  block packageSearch:
+    for d in myParentDirs(path):
+      if packageCache.hasKey(d):
+        #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name
+        return packageCache[d]
+      inc parents
+      for file in walkFiles(d / "*.nimble"):
+        result = file.splitFile.name
+        break packageSearch
+      for file in walkFiles(d / "*.babel"):
+        result = file.splitFile.name
+        break packageSearch
+  # we also store if we didn't find anything:
+  if result.isNil: result = ""
+  for d in myParentDirs(path):
+    #echo "set cache ", d, " |", result, "|", parents
+    packageCache[d] = result
+    dec parents
+    if parents <= 0: break
+
+proc withPackageName*(path: string): string =
+  let x = path.getPackageName
+  if x.len == 0:
+    result = path
+  else:
+    let (p, file, ext) = path.splitFile
+    result = (p / (x & '_' & file)) & ext