doc2 improvements; small lexer bugfix: backslashes in comments

compiler/commands.nim

@@ -268,6 +268,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "forcebuild", "f": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optForceFullMake)
+  of "project":
+    expectNoArg(switch, arg, pass, info)
+    gWholeProject = true
   of "gc": 
     expectArg(switch, arg, pass, info)
     case arg.normalize

compiler/docgen2.nim

@@ -23,6 +23,7 @@ type
 proc close(p: PPassContext, n: PNode): PNode =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
+  if gWholeProject or sfMainModule in g.module.flags:
     writeOutput(g.doc, g.filename, HtmlExt, useWarning)
     try:
       generateIndex(g.doc)

compiler/lexer.nim

@@ -679,6 +679,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
       inc(indent)
     if buf[pos] == '#' and (col == indent or lastBackslash > 0):
       tok.literal.add "\n"
+      col = indent
     else:
       if buf[pos] > ' ': 
         L.indentAhead = indent

compiler/main.nim

@@ -83,8 +83,18 @@ proc CompileModule(filename: string, flags: TSymFlags): PSym =
     result.id = getID()
   processModule(result, f, nil, rd)
 
+proc `==^`(a, b: string): bool =
+  try:
+    result = sameFile(a, b)
+  except EOS:
+    result = false
+
 proc CompileProject(projectFile = gProjectFull) =
-  discard CompileModule(options.libpath / "system", {sfSystemModule})
+  let systemFile = options.libpath / "system"
+  if projectFile.addFileExt(nimExt) ==^ systemFile.addFileExt(nimExt):
+    discard CompileModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    discard CompileModule(systemFile, {sfSystemModule})
     discard CompileModule(projectFile, {sfMainModule})
 
 proc semanticPasses =

compiler/options.nim

@@ -95,6 +95,8 @@ var
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
 
+  gWholeProject*: bool # for 'doc2': output any dependency
+  
 const 
   genSubDir* = "nimcache"
   NimExt* = "nim"

compiler/sempass2.nim

@@ -9,7 +9,7 @@
 
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees, 
-  wordrecg, strutils
+  wordrecg, strutils, options
 
 # Second semantic checking pass over the AST. Necessary because the old
 # way had some inherent problems. Performs:
@@ -92,30 +92,44 @@ proc excType(n: PNode): PType =
   let t = if n.kind == nkEmpty: sysTypeFromName"E_Base" else: n.typ
   result = skipTypes(t, skipPtrs)
 
+proc createRaise(n: PNode): PNode =
+  result = newNode(nkType)
+  result.typ = sysTypeFromName"E_Base"
+  if not n.isNil: result.info = n.info
+
+proc createTag(n: PNode): PNode =
+  result = newNode(nkType)
+  result.typ = sysTypeFromName"TEffect"
+  if not n.isNil: result.info = n.info
+
 proc addEffect(a: PEffects, e: PNode, useLineInfo=true) =
   assert e.kind != nkRaiseStmt
   var aa = a.exc
   for i in a.bottom .. <aa.len:
     if sameType(aa[i].excType, e.excType):
-      if not useLineInfo: return
+      if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
   throws(a.exc, e)
 
-proc mergeEffects(a: PEffects, b: PNode, useLineInfo: bool) =
-  if not b.isNil:
-    for effect in items(b): addEffect(a, effect, useLineInfo)
-
 proc addTag(a: PEffects, e: PNode, useLineInfo=true) =
   var aa = a.tags
   for i in 0 .. <aa.len:
     if sameType(aa[i].typ.skipTypes(skipPtrs), e.typ.skipTypes(skipPtrs)):
-      if not useLineInfo: return
+      if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
   throws(a.tags, e)
 
-proc mergeTags(a: PEffects, b: PNode, useLineInfo: bool) =
+proc mergeEffects(a: PEffects, b, comesFrom: PNode) =
-  if not b.isNil:
+  if b.isNil:
-    for effect in items(b): addTag(a, effect, useLineInfo)
+    addEffect(a, createRaise(comesFrom))
+  else:
+    for effect in items(b): addEffect(a, effect, useLineInfo=comesFrom != nil)
+
+proc mergeTags(a: PEffects, b, comesFrom: PNode) =
+  if b.isNil:
+    addTag(a, createTag(comesFrom))
+  else:
+    for effect in items(b): addTag(a, effect, useLineInfo=comesFrom != nil)
 
 proc listEffects(a: PEffects) =
   for e in items(a.exc):  Message(e.info, hintUser, typeToString(e.typ))
@@ -197,6 +211,8 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int) =
       var t = typeToString(real[i].typ)
       if t.startsWith("ref "): t = substr(t, 4)
       effects.sons[i] = newIdentNode(getIdent(t), n.info)
+      # set the type so that the following analysis doesn't screw up:
+      effects.sons[i].typ = real[i].typ
 
     var pair = newNode(nkExprColonExpr, n.info, @[
       newIdentNode(getIdent(specialWords[effectType]), n.info), effects])
@@ -208,30 +224,16 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int) =
 
 proc documentRaises*(n: PNode) =
   if n.sons[namePos].kind != nkSym: return
-
+  documentEffect(n, n.sons[pragmasPos], wRaises, exceptionEffects)
-  var x = n.sons[pragmasPos]
+  documentEffect(n, n.sons[pragmasPos], wTags, tagEffects)
-  documentEffect(n, x, wRaises, exceptionEffects)
-  documentEffect(n, x, wTags, tagEffects)
-
-proc createRaise(n: PNode): PNode =
-  result = newNodeIT(nkType, n.info, sysTypeFromName"E_Base")
-
-proc createTag(n: PNode): PNode =
-  result = newNodeIT(nkType, n.info, sysTypeFromName"TEffect")
 
 proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) =
   let pragma = s.ast.sons[pragmasPos]
   let spec = effectSpec(pragma, wRaises)
-  if not isNil(spec):
+  mergeEffects(tracked, spec, n)
-    mergeEffects(tracked, spec, useLineInfo=false)
-  else:
-    addEffect(tracked, createRaise(n))
   
   let tagSpec = effectSpec(pragma, wTags)
-  if not isNil(tagSpec):
+  mergeTags(tracked, tagSpec, n)
-    mergeTags(tracked, tagSpec, useLineInfo=false)
-  else:
-    addTag(tracked, createTag(n))
 
 proc track(tracked: PEffects, n: PNode) =
   case n.kind
@@ -254,8 +256,8 @@ proc track(tracked: PEffects, n: PNode) =
           addEffect(tracked, createRaise(n))
           addTag(tracked, createTag(n))
       else:
-        mergeEffects(tracked, effectList.sons[exceptionEffects], true)
+        mergeEffects(tracked, effectList.sons[exceptionEffects], n)
-        mergeTags(tracked, effectList.sons[tagEffects], true)
+        mergeTags(tracked, effectList.sons[tagEffects], n)
   of nkTryStmt:
     trackTryStmt(tracked, n)
     return
@@ -346,3 +348,4 @@ proc trackProc*(s: PSym, body: PNode) =
                     hints=off)
     # after the check, use the formal spec:
     effects.sons[tagEffects] = tagsSpec
+    

compiler/semstmts.nim

@@ -247,6 +247,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       if a.kind != nkVarTuple:
         v.typ = typ
         b = newNodeI(nkIdentDefs, a.info)
+        if gCmd == cmdDoc:
+          # keep documentation information:
+          b.comment = a.comment
         addSon(b, newSymNode(v))
         addSon(b, a.sons[length-2])      # keep type desc for doc generator
         addSon(b, copyTree(def))
@@ -284,6 +287,7 @@ proc semConst(c: PContext, n: PNode): PNode =
     v.ast = def               # no need to copy
     if sfGenSym notin v.flags: addInterfaceDecl(c, v)
     var b = newNodeI(nkConstDef, a.info)
+    if gCmd == cmdDoc: b.comment = a.comment
     addSon(b, newSymNode(v))
     addSon(b, ast.emptyNode)            # no type description
     addSon(b, copyTree(def))

compiler/transf.nim

@@ -150,6 +150,9 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
       newVar.owner = getCurrOwner(c)
       IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
       var defs = newTransNode(nkIdentDefs, it.info, 3)
+      if gCmd == cmdDoc:
+        # keep documentation information:
+        pnode(defs).comment = it.comment
       defs[0] = newSymNode(newVar).PTransNode
       defs[1] = it.sons[1].PTransNode
       defs[2] = transform(c, it.sons[2])
@@ -659,6 +662,11 @@ proc transform(c: PTransf, n: PNode): PTransNode =
       result = transformSons(c, n)
   of nkBlockStmt, nkBlockExpr:
     result = transformBlock(c, n)
+  of nkIdentDefs, nkConstDef:
+    result = transformSons(c, n)
+    # XXX comment handling really sucks:
+    if gCmd == cmdDoc:
+      pnode(result).comment = n.comment
   else:
     result = transformSons(c, n)
   var cnst = getConstExpr(c.module, PNode(result))

doc/advopt.txt

@@ -49,6 +49,7 @@ Advanced options:
                             (you should omit platform-specific extensions)
   --genMapping              generate a mapping file containing
                             (Nimrod, mangled) identifier pairs
+  --project                 document the whole project (doc2)
   --lineDir:on|off          generation of #line directive on|off
   --threadanalysis:on|off   turn thread analysis on|off
   --tlsEmulation:on|off     turn thread local storage emulation on|off

lib/system.nim

@@ -847,7 +847,12 @@ when taintMode:
   
   proc len*(s: TaintedString): int {.borrow.}
 else:
-  type TaintedString* = string
+  type TaintedString* = string          ## a distinct string type that 
+                                        ## is `tainted`:idx:. It is an alias for
+                                        ## ``string`` if the taint mode is not
+                                        ## turned on. Use the ``-d:taintMode``
+                                        ## command line switch to turn the taint
+                                        ## mode on.
 
 when defined(profiler):
   proc nimProfile() {.compilerProc, noinline.}
@@ -1817,8 +1822,7 @@ when not defined(EcmaScript) and not defined(NimrodVM):
     ## Returns true iff `f` is at the end.
     
   proc readChar*(f: TFile): char {.importc: "fgetc", nodecl, tags: [FReadIO].}
-    ## Reads a single character from the stream `f`. If the stream
+    ## Reads a single character from the stream `f`.
-    ## has no more characters, `EEndOfFile` is raised.
   proc FlushFile*(f: TFile) {.importc: "fflush", noDecl, tags: [FWriteIO].}
     ## Flushes `f`'s buffer.
 
@@ -2317,10 +2321,11 @@ proc raiseAssert*(msg: string) {.noinline.} =
   raise newException(EAssertionFailed, msg)
 
 when true:
-  proc hiddenRaiseAssert(msg: string) {.raises: [].} =
+  proc hiddenRaiseAssert(msg: string) {.raises: [], tags: [].} =
     # trick the compiler to not list ``EAssertionFailed`` when called
     # by ``assert``.
-    type THide = proc (msg: string) {.noinline, raises: [], noSideEffect.}
+    type THide = proc (msg: string) {.noinline, raises: [], noSideEffect,
+                                      tags: [].}
     THide(raiseAssert)(msg)
 
 template assert*(cond: bool, msg = "") =

todo.txt

@@ -1,11 +1,6 @@
 version 0.9.2
 =============
   
-- improve 'doc2': 
-  * respect output directory
-  * write only 1 file unless requested 'whole project'
-  * make it work for the 'system' module
-  * the documentation system should default to 'doc2'
 - test&finish first class iterators:
   * allow return in first class iterators
   * nested iterators
@@ -82,9 +77,7 @@ version 0.9.XX
 - macros need access to types and symbols (partially implemented)
 - document nimdoc properly finally
 - make 'clamp' a magic for the range stuff
-- implement a warning message for shadowed 'result' variable
 - we need to support iteration of 2 different data structures in parallel
-- implement proper coroutines
 - proc specialization in the code gen for write barrier specialization
 - tlastmod returns wrong results on BSD (Linux, MacOS X: works)
 - nested tuple unpacking; tuple unpacking in non-var-context

web/news.txt

@@ -27,6 +27,8 @@ Changes affecting backwards compatibility
 Compiler Additions
 ------------------
 
+- The ``doc2`` command does not generate output for the whole project anymore.
+  Use the new ``--project`` switch to enable this behaviour.
 - The compiler can now warn about shadowed local variables. However, this needs
   to be turned on explicitly via ``--warning[ShadowIdent]:on``.
 

web/nimrod.ini

@@ -25,28 +25,28 @@ file: ticker
 doc: "endb;intern;apis;lib;manual;tut1;tut2;nimrodc;overview;filters;trmacros"
 doc: "tools;c2nim;niminst;nimgrep;gc;estp"
 pdf: "manual;lib;tut1;tut2;nimrodc;c2nim;niminst;gc"
-srcdoc2: "impure/graphics;wrappers/sdl"
+srcdoc2: "system.nim;impure/graphics;wrappers/sdl"
-srcdoc: "core/macros;pure/marshal;core/typeinfo;core/unsigned"
+srcdoc2: "core/macros;pure/marshal;core/typeinfo;core/unsigned"
-srcdoc: "impure/re;pure/sockets"
+srcdoc2: "impure/re;pure/sockets"
-srcdoc: "system.nim;system/threads.nim;system/channels.nim"
+srcdoc: "system/threads.nim;system/channels.nim"
-srcdoc: "pure/os;pure/strutils;pure/math;pure/matchers;pure/algorithm"
+srcdoc2: "pure/os;pure/strutils;pure/math;pure/matchers;pure/algorithm"
-srcdoc: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib"
+srcdoc2: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib"
-srcdoc: "pure/parseopt;pure/hashes;pure/strtabs;pure/lexbase"
+srcdoc2: "pure/parseopt;pure/hashes;pure/strtabs;pure/lexbase"
-srcdoc: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql"
+srcdoc2: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql"
-srcdoc: "pure/streams;pure/terminal;pure/cgi;impure/web;pure/unicode"
+srcdoc2: "pure/streams;pure/terminal;pure/cgi;impure/web;pure/unicode"
-srcdoc: "impure/zipfiles;pure/htmlgen;pure/parseutils;pure/browsers"
+srcdoc2: "impure/zipfiles;pure/htmlgen;pure/parseutils;pure/browsers"
-srcdoc: "impure/db_postgres;impure/db_mysql;impure/db_sqlite;impure/db_mongo"
+srcdoc2: "impure/db_postgres;impure/db_mysql;impure/db_sqlite;impure/db_mongo"
-srcdoc: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl;pure/fsmonitor"
+srcdoc2: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl;pure/fsmonitor"
-srcdoc: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser"
+srcdoc2: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser"
-srcdoc: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes"
+srcdoc2: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes"
-srcdoc: "pure/json;pure/base64;pure/scgi;pure/redis;impure/graphics"
+srcdoc2: "pure/json;pure/base64;pure/scgi;pure/redis;impure/graphics"
-srcdoc: "impure/rdstdin;wrappers/zmq;wrappers/sphinx"
+srcdoc2: "impure/rdstdin;wrappers/zmq;wrappers/sphinx"
-srcdoc: "pure/collections/tables;pure/collections/sets;pure/collections/lists"
+srcdoc2: "pure/collections/tables;pure/collections/sets;pure/collections/lists"
-srcdoc: "pure/collections/intsets;pure/collections/queues;pure/encodings"
+srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/encodings"
-srcdoc: "pure/events;pure/collections/sequtils;pure/irc;ecmas/dom;pure/cookies"
+srcdoc2: "pure/events;pure/collections/sequtils;pure/irc;ecmas/dom;pure/cookies"
-srcdoc: "pure/ftpclient;pure/memfiles;pure/subexes;pure/collections/critbits"
+srcdoc2: "pure/ftpclient;pure/memfiles;pure/subexes;pure/collections/critbits"
-srcdoc: "pure/asyncio;pure/actors;core/locks;pure/oids;pure/endians;pure/uri"
+srcdoc2: "pure/asyncio;pure/actors;core/locks;pure/oids;pure/endians;pure/uri"
-srcdoc: "pure/nimprof"
+srcdoc2: "pure/nimprof"
 
 webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup"
 webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc"

web/question.txt

@@ -72,7 +72,7 @@ statements.
 How fast is Nimrod?
 -------------------
 Benchmarks show it to be comparable to C. Some language features (methods, 
-closures, RTTI) are not yet as optimized as they could and will be.
+closures, message passing) are not yet as optimized as they could and will be.
 The only overhead Nimrod has over C is the GC which has been tuned 
 for years but still needs some work.