fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting; other bug fix (#14439)

* fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting
* remove orig deadcode from getAllRunnableExamplesImpl
* fix expected examples
* add test to close https://github.com/nim-lang/Nim/issues/14473
* correctly handle regular comments before 1st token inside runnableExamples
* add test to answer https://github.com/nim-lang/Nim/pull/14439#discussion_r431829199
* update tests

nimdoc/testproject/testproject.nim

@@ -9,17 +9,22 @@ runnableExamples:
   # bug #11078
   for x in "xx": discard
 
+when true:
+  ## top2
+  runnableExamples:
+    discard "in top2"
+  ## top2 after
+
+runnableExamples:
+  discard "in top3"
+## top3 after
+
 const
   C_A* = 0x7FF0000000000000'f64
   C_B* = 0o377'i8
   C_C* = 0o277'i8
   C_D* = 0o177777'i16
 
-template foo*(a, b: SomeType) =
-  ## This does nothing
-  ##
-  discard
-
 proc bar*[T](a, b: T): T =
   result = a + b
 
@@ -33,13 +38,207 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} =
 
 import std/macros
 
-macro bar*(): untyped =
-  result = newStmtList()
-
 var aVariable*: array[1, int]
 
+# bug #9432
 aEnum()
 bEnum()
+fromUtilsGen()
 
-# bug #9432
 proc isValid*[T](x: T): bool = x.len > 0
+
+when true:
+  # these cases appear redundant but they're actually (almost) all different at
+  # AST level and needed to ensure docgen keeps working, eg because of issues
+  # like D20200526T163511
+  type
+    Foo* = enum
+      enumValueA2
+
+  proc z1*(): Foo =
+    ## cz1
+    Foo.default
+
+  proc z2*() =
+    ## cz2
+    runnableExamples:
+      discard "in cz2"
+
+  proc z3*() =
+    ## cz3
+
+  proc z4*() =
+    ## cz4
+    discard
+
+when true:
+  # tests for D20200526T163511
+  proc z5*(): int =
+    ## cz5
+    return 1
+
+  proc z6*(): int =
+    ## cz6
+    1
+
+  template z6t*(): int =
+    ## cz6t
+    1
+
+  proc z7*(): int =
+    ## cz7
+    result = 1
+
+  proc z8*(): int =
+    ## cz8
+    block:
+      discard
+      1+1
+
+when true:
+  # interleaving 0 or more runnableExamples and doc comments, issue #9227
+  proc z9*() =
+    runnableExamples: doAssert 1 + 1 == 2
+
+  proc z10*() =
+    runnableExamples "-d:foobar":
+      discard 1
+    ## cz10
+
+  proc z11*() =
+    runnableExamples:
+      discard 1
+    discard
+
+  proc z12*(): int =
+    runnableExamples:
+      discard 1
+    12
+
+  proc z13*() =
+    ## cz13
+    runnableExamples:
+      discard
+
+  proc baz*() = discard
+
+  proc bazNonExported() =
+    ## out (not exported)
+    runnableExamples:
+      # BUG: this currently this won't be run since not exported
+      # but probably should
+      doAssert false
+
+  proc z17*() =
+    # BUG: a comment before 1st doc comment currently doesn't prevent
+    # doc comment from being docgen'd; probably should be fixed
+    ## cz17
+    ## rest
+    runnableExamples:
+      discard 1
+    ## rest
+    # this comment separates docgen'd doc comments
+    ## out
+
+when true: # capture non-doc comments correctly even before 1st token
+  proc p1*() =
+    ## cp1
+    runnableExamples: doAssert 1 == 1 # regular comments work here
+    ## c4
+    runnableExamples:
+      # c5 regular comments before 1st token work
+      # regular comment
+      #[
+      nested regular comment
+      ]#
+      doAssert 2 == 2 # c8
+      ## this is a non-nested doc comment
+
+      ##[
+      this is a nested doc comment
+      ]##
+      discard "c9"
+      # also work after
+    # this should be out
+
+when true: # (most) macros
+  macro bar*(): untyped =
+    result = newStmtList()
+
+  macro z16*() =
+    runnableExamples: discard 1
+    ## cz16
+    ## after
+    runnableExamples:
+      doAssert 2 == 1 + 1
+    # BUG: we should probably render `cz16\nafter` by keeping newline instead or
+    # what it currently renders as: `cz16 after`
+
+  macro z18*(): int =
+    ## cz18
+    newLit 0
+
+when true: # (most) templates
+  template foo*(a, b: SomeType) =
+    ## This does nothing
+    ##
+    discard
+
+  template myfn*() =
+    runnableExamples:
+      import std/strutils
+      ## issue #8871 preserve formatting
+      ## line doc comment
+      # bar
+      doAssert "'foo" == "'foo"
+      ##[
+      foo
+      bar
+      ]##
+
+      doAssert: not "foo".startsWith "ba"
+      block:
+        discard 0xff # elu par cette crapule
+      # should be in
+    ## should be still in
+
+    # out
+    ## out
+
+  template z14*() =
+    ## cz14
+    runnableExamples:
+      discard
+
+  template z15*() =
+    ## cz15
+    runnableExamples:
+      discard
+    runnableExamples: discard 3
+    runnableExamples: discard 4
+    ## ok5
+    ## ok5b
+    runnableExamples: assert true
+
+    ## in or out?
+    # this is an edge case; a newline separate last runnableExamples from 
+    # next doc comment but AST isnt' aware of it; this could change in future
+    discard 8
+    ## out
+    runnableExamples: discard 1
+
+when true: # issue #14473
+  import std/[sequtils]
+  template doit(): untyped =
+    ## doit
+    ## return output only
+    toSeq([1,2])
+  echo doit() # using doAssert or similar to avoid echo would "hide" the original bug
+
+when true:
+  template testNimDocTrailingExample*() =
+    # this must be last entry in this file, it checks against a bug (that got fixed)
+    # where runnableExamples would not show if there was not at least 2 "\n" after
+    # the last character of runnableExamples
+    runnableExamples:
+      discard 2