doc2 improvements; small lexer bugfix: backslashes in comments

compiler/commands.nim

@@ -268,6 +268,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "forcebuild", "f": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optForceFullMake)
+  of "project":
+    expectNoArg(switch, arg, pass, info)
+    gWholeProject = true
   of "gc": 
     expectArg(switch, arg, pass, info)
     case arg.normalize

compiler/docgen2.nim

@@ -23,11 +23,12 @@ type
 proc close(p: PPassContext, n: PNode): PNode =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
-  writeOutput(g.doc, g.filename, HtmlExt, useWarning)
-  try:
-    generateIndex(g.doc)
-  except EIO:
-    nil
+  if gWholeProject or sfMainModule in g.module.flags:
+    writeOutput(g.doc, g.filename, HtmlExt, useWarning)
+    try:
+      generateIndex(g.doc)
+    except EIO:
+      nil
 
 proc processNode(c: PPassContext, n: PNode): PNode = 
   result = n

compiler/lexer.nim

@@ -679,6 +679,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
       inc(indent)
     if buf[pos] == '#' and (col == indent or lastBackslash > 0):
       tok.literal.add "\n"
+      col = indent
     else:
       if buf[pos] > ' ': 
         L.indentAhead = indent

compiler/main.nim

@@ -83,9 +83,19 @@ proc CompileModule(filename: string, flags: TSymFlags): PSym =
     result.id = getID()
   processModule(result, f, nil, rd)
 
+proc `==^`(a, b: string): bool =
+  try:
+    result = sameFile(a, b)
+  except EOS:
+    result = false
+
 proc CompileProject(projectFile = gProjectFull) =
-  discard CompileModule(options.libpath / "system", {sfSystemModule})
-  discard CompileModule(projectFile, {sfMainModule})
+  let systemFile = options.libpath / "system"
+  if projectFile.addFileExt(nimExt) ==^ systemFile.addFileExt(nimExt):
+    discard CompileModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    discard CompileModule(systemFile, {sfSystemModule})
+    discard CompileModule(projectFile, {sfMainModule})
 
 proc semanticPasses =
   registerPass(verbosePass())

compiler/options.nim

@@ -95,6 +95,8 @@ var
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
 
+  gWholeProject*: bool # for 'doc2': output any dependency
+  
 const 
   genSubDir* = "nimcache"
   NimExt* = "nim"

compiler/sempass2.nim

@@ -9,7 +9,7 @@
 
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees, 
-  wordrecg, strutils
+  wordrecg, strutils, options
 
 # Second semantic checking pass over the AST. Necessary because the old
 # way had some inherent problems. Performs:
@@ -92,30 +92,44 @@ proc excType(n: PNode): PType =
   let t = if n.kind == nkEmpty: sysTypeFromName"E_Base" else: n.typ
   result = skipTypes(t, skipPtrs)
 
+proc createRaise(n: PNode): PNode =
+  result = newNode(nkType)
+  result.typ = sysTypeFromName"E_Base"
+  if not n.isNil: result.info = n.info
+
+proc createTag(n: PNode): PNode =
+  result = newNode(nkType)
+  result.typ = sysTypeFromName"TEffect"
+  if not n.isNil: result.info = n.info
+
 proc addEffect(a: PEffects, e: PNode, useLineInfo=true) =
   assert e.kind != nkRaiseStmt
   var aa = a.exc
   for i in a.bottom .. <aa.len:
     if sameType(aa[i].excType, e.excType):
-      if not useLineInfo: return
+      if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
   throws(a.exc, e)
 
-proc mergeEffects(a: PEffects, b: PNode, useLineInfo: bool) =
-  if not b.isNil:
-    for effect in items(b): addEffect(a, effect, useLineInfo)
-
 proc addTag(a: PEffects, e: PNode, useLineInfo=true) =
   var aa = a.tags
   for i in 0 .. <aa.len:
     if sameType(aa[i].typ.skipTypes(skipPtrs), e.typ.skipTypes(skipPtrs)):
-      if not useLineInfo: return
+      if not useLineInfo or gCmd == cmdDoc: return
       elif aa[i].info == e.info: return
   throws(a.tags, e)
 
-proc mergeTags(a: PEffects, b: PNode, useLineInfo: bool) =
-  if not b.isNil:
-    for effect in items(b): addTag(a, effect, useLineInfo)
+proc mergeEffects(a: PEffects, b, comesFrom: PNode) =
+  if b.isNil:
+    addEffect(a, createRaise(comesFrom))
+  else:
+    for effect in items(b): addEffect(a, effect, useLineInfo=comesFrom != nil)
+
+proc mergeTags(a: PEffects, b, comesFrom: PNode) =
+  if b.isNil:
+    addTag(a, createTag(comesFrom))
+  else:
+    for effect in items(b): addTag(a, effect, useLineInfo=comesFrom != nil)
 
 proc listEffects(a: PEffects) =
   for e in items(a.exc):  Message(e.info, hintUser, typeToString(e.typ))
@@ -197,6 +211,8 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int) =
       var t = typeToString(real[i].typ)
       if t.startsWith("ref "): t = substr(t, 4)
       effects.sons[i] = newIdentNode(getIdent(t), n.info)
+      # set the type so that the following analysis doesn't screw up:
+      effects.sons[i].typ = real[i].typ
 
     var pair = newNode(nkExprColonExpr, n.info, @[
       newIdentNode(getIdent(specialWords[effectType]), n.info), effects])
@@ -208,34 +224,20 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int) =
 
 proc documentRaises*(n: PNode) =
   if n.sons[namePos].kind != nkSym: return
-
-  var x = n.sons[pragmasPos]
-  documentEffect(n, x, wRaises, exceptionEffects)
-  documentEffect(n, x, wTags, tagEffects)
-
-proc createRaise(n: PNode): PNode =
-  result = newNodeIT(nkType, n.info, sysTypeFromName"E_Base")
-
-proc createTag(n: PNode): PNode =
-  result = newNodeIT(nkType, n.info, sysTypeFromName"TEffect")
+  documentEffect(n, n.sons[pragmasPos], wRaises, exceptionEffects)
+  documentEffect(n, n.sons[pragmasPos], wTags, tagEffects)
 
 proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) =
   let pragma = s.ast.sons[pragmasPos]
   let spec = effectSpec(pragma, wRaises)
-  if not isNil(spec):
-    mergeEffects(tracked, spec, useLineInfo=false)
-  else:
-    addEffect(tracked, createRaise(n))
+  mergeEffects(tracked, spec, n)
   
   let tagSpec = effectSpec(pragma, wTags)
-  if not isNil(tagSpec):
-    mergeTags(tracked, tagSpec, useLineInfo=false)
-  else:
-    addTag(tracked, createTag(n))
+  mergeTags(tracked, tagSpec, n)
 
 proc track(tracked: PEffects, n: PNode) =
   case n.kind
-  of nkRaiseStmt: 
+  of nkRaiseStmt:
     n.sons[0].info = n.info
     throws(tracked.exc, n.sons[0])
   of nkCallKinds:
@@ -254,8 +256,8 @@ proc track(tracked: PEffects, n: PNode) =
           addEffect(tracked, createRaise(n))
           addTag(tracked, createTag(n))
       else:
-        mergeEffects(tracked, effectList.sons[exceptionEffects], true)
-        mergeTags(tracked, effectList.sons[tagEffects], true)
+        mergeEffects(tracked, effectList.sons[exceptionEffects], n)
+        mergeTags(tracked, effectList.sons[tagEffects], n)
   of nkTryStmt:
     trackTryStmt(tracked, n)
     return
@@ -346,3 +348,4 @@ proc trackProc*(s: PSym, body: PNode) =
                     hints=off)
     # after the check, use the formal spec:
     effects.sons[tagEffects] = tagsSpec
+    

compiler/semstmts.nim

@@ -247,6 +247,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       if a.kind != nkVarTuple:
         v.typ = typ
         b = newNodeI(nkIdentDefs, a.info)
+        if gCmd == cmdDoc:
+          # keep documentation information:
+          b.comment = a.comment
         addSon(b, newSymNode(v))
         addSon(b, a.sons[length-2])      # keep type desc for doc generator
         addSon(b, copyTree(def))
@@ -284,6 +287,7 @@ proc semConst(c: PContext, n: PNode): PNode =
     v.ast = def               # no need to copy
     if sfGenSym notin v.flags: addInterfaceDecl(c, v)
     var b = newNodeI(nkConstDef, a.info)
+    if gCmd == cmdDoc: b.comment = a.comment
     addSon(b, newSymNode(v))
     addSon(b, ast.emptyNode)            # no type description
     addSon(b, copyTree(def))

compiler/transf.nim

@@ -150,6 +150,9 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
       newVar.owner = getCurrOwner(c)
       IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
       var defs = newTransNode(nkIdentDefs, it.info, 3)
+      if gCmd == cmdDoc:
+        # keep documentation information:
+        pnode(defs).comment = it.comment
       defs[0] = newSymNode(newVar).PTransNode
       defs[1] = it.sons[1].PTransNode
       defs[2] = transform(c, it.sons[2])
@@ -659,6 +662,11 @@ proc transform(c: PTransf, n: PNode): PTransNode =
       result = transformSons(c, n)
   of nkBlockStmt, nkBlockExpr:
     result = transformBlock(c, n)
+  of nkIdentDefs, nkConstDef:
+    result = transformSons(c, n)
+    # XXX comment handling really sucks:
+    if gCmd == cmdDoc:
+      pnode(result).comment = n.comment
   else:
     result = transformSons(c, n)
   var cnst = getConstExpr(c.module, PNode(result))