integrate 'koch web' features into koch; deprecate 'nimweb' tool

doc/tools.rst

@@ -5,7 +5,7 @@ Tools available with Nim
 The standard distribution ships with the following tools:
 
 - | `Documentation generator <docgen.html>`_
-  | The builtin document generator ``nim doc2`` generates HTML documentation
+  | The builtin document generator ``nim doc`` generates HTML documentation
     from ``.nim`` source files.
 
 - | `Nimsuggest for IDE support <nimsuggest.html>`_
@@ -13,17 +13,8 @@ The standard distribution ships with the following tools:
     and obtain useful information like definition of symbols or suggestions for
     completion.
 
-- | `Nim Installation Generator <niminst.html>`_
-  | How to generate a nice installer for your Nim program.
-
 - | `C2nim <c2nim.html>`_
   | C to Nim source converter. Translates C header files to Nim.
 
 - | `nimgrep <nimgrep.html>`_
   | Nim search and replace utility.
-
-- | `endb <endb.html>`_
-  | Nim's slow platform independent embedded debugger.
-
-- | `estp <estp.html>`_
-  | Nim's slow platform independent embedded stack trace profiler.

koch.nim

@@ -23,6 +23,8 @@ when defined(i386) and defined(windows) and defined(vcc):
 import
   os, strutils, parseopt, osproc, streams
 
+import tools / kochdocs
+
 const VersionAsString = system.NimVersion
 
 const
@@ -49,9 +51,7 @@ Boot options:
                            (not needed on Windows)
 
 Commands for core developers:
-  web [options]            generates the website and the full documentation
+  docs [options]           generates the full documentation
-                           (see `nimweb.nim` for cmd line options)
-  website [options]        generates only the website
   csource -d:release       builds the C sources for installation
   pdf                      builds the PDF documentation
   zip                      builds the installation zip package
@@ -67,13 +67,6 @@ Web options:
                            build the official docs, use UA-48159761-1
 """
 
-const gaCode = " --googleAnalytics:UA-48159761-1"
-
-proc exe(f: string): string =
-  result = addFileExt(f, ExeExt)
-  when defined(windows):
-    result = result.replace('/','\\')
-
 template withDir(dir, body) =
   let old = getCurrentDir()
   try:
@@ -82,43 +75,6 @@ template withDir(dir, body) =
   finally:
     setCurrentdir(old)
 
-proc findNim(): string =
-  var nim = "nim".exe
-  result = "bin" / nim
-  if existsFile(result): return
-  for dir in split(getEnv("PATH"), PathSep):
-    if existsFile(dir / nim): return dir / nim
-  # assume there is a symlink to the exe or something:
-  return nim
-
-proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
-  let prevPath = getEnv("PATH")
-  if additionalPath.len > 0:
-    var absolute = additionalPATH
-    if not absolute.isAbsolute:
-      absolute = getCurrentDir() / absolute
-    echo("Adding to $PATH: ", absolute)
-    putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute)
-  echo(cmd)
-  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
-  putEnv("PATH", prevPath)
-
-proc nimexec(cmd: string) =
-  exec findNim() & " " & cmd
-
-proc execCleanPath(cmd: string,
-                   additionalPath = ""; errorcode: int = QuitFailure) =
-  # simulate a poor man's virtual environment
-  let prevPath = getEnv("PATH")
-  when defined(windows):
-    let CleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT"
-  else:
-    const CleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin"
-  putEnv("PATH", CleanPath & PathSep & additionalPath)
-  echo(cmd)
-  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
-  putEnv("PATH", prevPath)
-
 proc testUnixInstall() =
   let oldCurrentDir = getCurrentDir()
   try:
@@ -136,7 +92,7 @@ proc testUnixInstall() =
       execCleanPath("./bin/nim c koch.nim")
       execCleanPath("./koch boot -d:release", destDir / "bin")
       # check the docs build:
-      execCleanPath("./koch web", destDir / "bin")
+      execCleanPath("./koch docs", destDir / "bin")
       # check nimble builds:
       execCleanPath("./koch tools")
       # check the tests work:
@@ -295,18 +251,19 @@ proc install(args: string) =
   geninstall()
   exec("sh ./install.sh $#" % args)
 
-proc web(args: string) =
+when false:
-  nimexec("js tools/dochack/dochack.nim")
+  proc web(args: string) =
-  nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" %
+    nimexec("js tools/dochack/dochack.nim")
-       [args, VersionAsString])
+    nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" %
+        [args, VersionAsString])
 
-proc website(args: string) =
+  proc website(args: string) =
-  nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" %
+    nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" %
-       [args, VersionAsString])
+        [args, VersionAsString])
 
-proc pdf(args="") =
+  proc pdf(args="") =
-  exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" %
+    exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" %
-       [findNim(), args, VersionAsString], additionalPATH=findNim().splitFile.dir)
+        [findNim(), args, VersionAsString], additionalPATH=findNim().splitFile.dir)
 
 # -------------- boot ---------------------------------------------------------
 
@@ -429,7 +386,7 @@ proc winRelease*() =
   # anymore!
   # Build -docs file:
   when true:
-    web(gaCode)
+    buildDocs(gaCode)
     withDir "web/upload/" & VersionAsString:
       exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
     overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString,
@@ -553,14 +510,11 @@ when isMainModule:
     case normalize(op.key)
     of "boot": boot(op.cmdLineRest)
     of "clean": clean(op.cmdLineRest)
-    of "web": web(op.cmdLineRest)
+    of "doc", "docs": buildDocs(op.cmdLineRest)
-    of "doc", "docs": web("--onlyDocs " & op.cmdLineRest)
+    of "doc0", "docs0":
-    of "json2": web("--json2 " & op.cmdLineRest)
-    of "website": website(op.cmdLineRest & gaCode)
-    of "web0":
       # undocumented command for Araq-the-merciful:
-      web(op.cmdLineRest & gaCode)
+      buildDocs(op.cmdLineRest & gaCode)
-    of "pdf": pdf()
+    of "pdf": buildPdfDoc(op.cmdLineRest, "doc/pdf")
     of "csource", "csources": csource(op.cmdLineRest)
     of "zip": zip(op.cmdLineRest)
     of "xz": xz(op.cmdLineRest)

tools/kochdocs.nim

@@ -0,0 +1,331 @@
+## Part of 'koch' responsible for the documentation generation.
+
+import os, strutils, osproc
+
+const
+  gaCode* = " --googleAnalytics:UA-48159761-1"
+
+  nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot --putenv:nimversion=$#" % system.NimVersion
+  gitUrl = "https://github.com/nim-lang/Nim"
+  docHtmlOutput = "doc/html"
+  webUploadOutput = "web/upload"
+
+proc exe*(f: string): string =
+  result = addFileExt(f, ExeExt)
+  when defined(windows):
+    result = result.replace('/','\\')
+
+proc findNim*(): string =
+  var nim = "nim".exe
+  result = "bin" / nim
+  if existsFile(result): return
+  for dir in split(getEnv("PATH"), PathSep):
+    if existsFile(dir / nim): return dir / nim
+  # assume there is a symlink to the exe or something:
+  return nim
+
+
+proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
+  let prevPath = getEnv("PATH")
+  if additionalPath.len > 0:
+    var absolute = additionalPATH
+    if not absolute.isAbsolute:
+      absolute = getCurrentDir() / absolute
+    echo("Adding to $PATH: ", absolute)
+    putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute)
+  echo(cmd)
+  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
+  putEnv("PATH", prevPath)
+
+proc execCleanPath*(cmd: string,
+                   additionalPath = ""; errorcode: int = QuitFailure) =
+  # simulate a poor man's virtual environment
+  let prevPath = getEnv("PATH")
+  when defined(windows):
+    let CleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT"
+  else:
+    const CleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin"
+  putEnv("PATH", CleanPath & PathSep & additionalPath)
+  echo(cmd)
+  if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
+  putEnv("PATH", prevPath)
+
+proc nimexec*(cmd: string) =
+  exec findNim() & " " & cmd
+
+const
+  pdf = """
+doc/manual.rst
+doc/lib.rst
+doc/tut1.rst
+doc/tut2.rst
+doc/nimc.rst
+doc/niminst.rst
+doc/gc.rst
+""".splitWhitespace()
+
+  rst2html = """
+doc/intern.rst
+doc/apis.rst
+doc/lib.rst
+doc/manual.rst
+doc/tut1.rst
+doc/tut2.rst
+doc/nimc.rst
+doc/overview.rst
+doc/filters.rst
+doc/tools.rst
+doc/niminst.rst
+doc/nimgrep.rst
+doc/gc.rst
+doc/estp.rst
+doc/idetools.rst
+doc/docgen.rst
+doc/koch.rst
+doc/backends.rst
+doc/nimsuggest.rst
+doc/nep1.rst
+doc/nims.rst
+doc/contributing.rst
+doc/manual/var_t_return.rst
+""".splitWhitespace()
+
+  doc = """
+lib/system.nim
+lib/system/nimscript.nim
+lib/pure/ospaths.nim
+lib/core/macros.nim
+lib/pure/marshal.nim
+lib/core/typeinfo.nim
+lib/impure/re.nim
+lib/pure/typetraits.nim
+nimsuggest/sexp.nim
+lib/pure/concurrency/threadpool.nim
+lib/pure/concurrency/cpuinfo.nim
+lib/js/dom.nim
+lib/js/jsffi.nim
+lib/js/jsconsole.nim
+lib/js/asyncjs.nim
+lib/pure/os.nim
+lib/pure/strutils.nim
+lib/pure/math.nim
+lib/pure/matchers.nim
+lib/pure/algorithm.nim
+lib/pure/stats.nim
+lib/windows/winlean.nim
+lib/pure/random.nim
+lib/pure/complex.nim
+lib/pure/times.nim
+lib/pure/osproc.nim
+lib/pure/pegs.nim
+lib/pure/dynlib.nim
+lib/pure/strscans.nim
+lib/pure/parseopt.nim
+lib/pure/hashes.nim
+lib/pure/strtabs.nim
+lib/pure/lexbase.nim
+lib/pure/parsecfg.nim
+lib/pure/parsexml.nim
+lib/pure/parsecsv.nim
+lib/pure/parsesql.nim
+lib/pure/streams.nim
+lib/pure/terminal.nim
+lib/pure/cgi.nim
+lib/pure/unicode.nim
+lib/pure/strmisc.nim
+lib/pure/htmlgen.nim
+lib/pure/parseutils.nim
+lib/pure/browsers.nim
+lib/impure/db_postgres.nim
+lib/impure/db_mysql.nim
+lib/impure/db_sqlite.nim
+lib/pure/db_common.nim
+lib/pure/httpserver.nim
+lib/pure/httpclient.nim
+lib/pure/smtp.nim
+lib/impure/ssl.nim
+lib/pure/ropes.nim
+lib/pure/unidecode/unidecode.nim
+lib/pure/xmldom.nim
+lib/pure/xmldomparser.nim
+lib/pure/xmlparser.nim
+lib/pure/htmlparser.nim
+lib/pure/xmltree.nim
+lib/pure/colors.nim
+lib/pure/mimetypes.nim
+lib/pure/json.nim
+lib/pure/base64.nim
+lib/pure/scgi.nim
+lib/pure/collections/tables.nim
+lib/pure/collections/sets.nim
+lib/pure/collections/lists.nim
+lib/pure/collections/sharedlist.nim
+lib/pure/collections/sharedtables.nim
+lib/pure/collections/intsets.nim
+lib/pure/collections/queues.nim
+lib/pure/collections/deques.nim
+lib/pure/encodings.nim
+lib/pure/collections/sequtils.nim
+lib/pure/cookies.nim
+lib/pure/memfiles.nim
+lib/pure/subexes.nim
+lib/pure/collections/critbits.nim
+lib/core/locks.nim
+lib/core/rlocks.nim
+lib/pure/oids.nim
+lib/pure/endians.nim
+lib/pure/uri.nim
+lib/pure/nimprof.nim
+lib/pure/unittest.nim
+lib/packages/docutils/highlite.nim
+lib/packages/docutils/rst.nim
+lib/packages/docutils/rstast.nim
+lib/packages/docutils/rstgen.nim
+lib/pure/logging.nim
+lib/pure/options.nim
+lib/pure/asyncdispatch.nim
+lib/pure/asyncnet.nim
+lib/pure/asyncstreams.nim
+lib/pure/asyncfutures.nim
+lib/pure/nativesockets.nim
+lib/pure/asynchttpserver.nim
+lib/pure/net.nim
+lib/pure/selectors.nim
+lib/pure/sugar.nim
+lib/pure/collections/chains.nim
+lib/pure/asyncfile.nim
+lib/pure/asyncftpclient.nim
+lib/pure/lenientops.nim
+lib/pure/md5.nim
+lib/pure/rationals.nim
+lib/pure/distros.nim
+lib/pure/oswalkdir.nim
+lib/pure/collections/heapqueue.nim
+lib/pure/fenv.nim
+lib/std/sha1.nim
+lib/impure/rdstdin.nim
+lib/pure/strformat.nim
+lib/pure/segfaults.nim
+lib/pure/mersenne.nim
+lib/pure/coro.nim
+lib/pure/httpcore.nim
+lib/pure/bitops.nim
+lib/pure/nimtracker.nim
+lib/pure/punycode.nim
+lib/pure/volatile.nim
+""".splitWhitespace()
+
+  doc0 = """
+lib/system/threads.nim
+lib/system/channels.nim
+""".splitWhitespace()
+
+  withoutIndex = """
+lib/wrappers/mysql.nim
+lib/wrappers/iup.nim
+lib/wrappers/sqlite3.nim
+lib/wrappers/postgres.nim
+lib/wrappers/tinyc.nim
+lib/wrappers/odbcsql.nim
+lib/wrappers/pcre.nim
+lib/wrappers/openssl.nim
+lib/posix/posix.nim
+lib/wrappers/odbcsql.nim
+lib/js/jscore.nim
+""".splitWhitespace()
+
+proc sexec(cmds: openarray[string]) =
+  ## Serial queue wrapper around exec.
+  for cmd in cmds:
+    echo(cmd)
+    let (outp, exitCode) = osproc.execCmdEx(cmd)
+    if exitCode != 0: quit outp
+
+proc mexec(cmds: openarray[string]) =
+  ## Multiprocessor version of exec
+  let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd})
+  if r != 0:
+    echo "external program failed, retrying serial work queue for logs!"
+    sexec(cmds)
+
+proc buildDocSamples(nimArgs, destPath: string) =
+  ## Special case documentation sample proc.
+  ##
+  ## TODO: consider integrating into the existing generic documentation builders
+  ## now that we have a single `doc` command.
+  exec(findNim() & " doc $# -o:$# $#" %
+    [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"])
+
+proc pathPart(d: string): string = splitFile(d).dir.replace('\\', '/')
+
+proc buildDoc(nimArgs, destPath: string) =
+  # call nim for the documentation:
+  var
+    commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len)
+    i = 0
+  let nim = findNim()
+  for d in items(rst2html):
+    commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" %
+      [nimArgs, gitUrl,
+      destPath / changeFileExt(splitFile(d).name, "html"), d]
+    i.inc
+  for d in items(doc0):
+    commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" %
+      [nimArgs, gitUrl,
+      destPath / changeFileExt(splitFile(d).name, "html"), d]
+    i.inc
+  for d in items(doc):
+    commands[i] = nim & " doc $# --git.url:$# -o:$# --index:on $#" %
+      [nimArgs, gitUrl,
+      destPath / changeFileExt(splitFile(d).name, "html"), d]
+    i.inc
+  for d in items(withoutIndex):
+    commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" %
+      [nimArgs, gitUrl,
+      destPath / changeFileExt(splitFile(d).name, "html"), d]
+    i.inc
+
+  mexec(commands)
+  exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath])
+
+proc buildPdfDoc*(nimArgs, destPath: string) =
+  createDir(destPath)
+  if os.execShellCmd("pdflatex -version") != 0:
+    echo "pdflatex not found; no PDF documentation generated"
+  else:
+    const pdflatexcmd = "pdflatex -interaction=nonstopmode "
+    for d in items(pdf):
+      exec(findNim() & " rst2tex $# $#" % [nimArgs, d])
+      # call LaTeX twice to get cross references right:
+      exec(pdflatexcmd & changeFileExt(d, "tex"))
+      exec(pdflatexcmd & changeFileExt(d, "tex"))
+      # delete all the crappy temporary files:
+      let pdf = splitFile(d).name & ".pdf"
+      let dest = destPath / pdf
+      removeFile(dest)
+      moveFile(dest=dest, source=pdf)
+      removeFile(changeFileExt(pdf, "aux"))
+      if existsFile(changeFileExt(pdf, "toc")):
+        removeFile(changeFileExt(pdf, "toc"))
+      removeFile(changeFileExt(pdf, "log"))
+      removeFile(changeFileExt(pdf, "out"))
+      removeFile(changeFileExt(d, "tex"))
+
+proc buildJS() =
+  exec(findNim() & " js -d:release --out:$1 web/nimblepkglist.nim" %
+      [webUploadOutput / "nimblepkglist.js"])
+  exec(findNim() & " js tools/dochack/dochack.nim")
+
+proc buildDocs*(args: string) =
+  let a = nimArgs & " " & args
+  buildJS()
+  let docup = webUploadOutput / NimVersion
+  createDir(docup)
+  buildDocSamples(a, docup)
+  buildDoc(a, docup)
+
+  # 'nimArgs' instead of 'a' is correct here because we don't want
+  # that the offline docs contain the 'gaCode'!
+  createDir(docHtmlOutput)
+  buildDocSamples(nimArgs, docHtmlOutput)
+  buildDoc(nimArgs, docHtmlOutput)