adds an option to interleave the generated code with snippets from the original source

Lines from the original source are outputted as comments next to line directives.
Hopefully, this will make debugging codegen problems easier.

Other changes:
The frame setup code now uses a single-line C macro. My motivation was to reduce
the noise in the generated output and make it easier to step over the boiler-plate
code, but counter-intuitively this also improved the overall compilation speed a
little bit so I applied the same treatment to line tracking too (this reduces the size of
the generated files and the explanation is that probably the I/O overhead dominates
the macro expansion costs).

compiler/ccgutils.nim

@@ -154,37 +154,12 @@ proc TableGetType*(tab: TIdTable, key: PType): PObject =
         if sameType(t, key): 
           return tab.data[h].val
 
-proc toCChar*(c: Char): string = 
-  case c
-  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
-  of '\'', '\"', '\\': result = '\\' & c
-  else: result = $(c)
-
 proc makeSingleLineCString*(s: string): string =
   result = "\""
   for c in items(s):
     result.add(c.toCChar)
   result.add('\"')
 
-proc makeCString*(s: string): PRope = 
-  # BUGFIX: We have to split long strings into many ropes. Otherwise
-  # this could trigger an InternalError(). See the ropes module for
-  # further information.
-  const 
-    MaxLineLength = 64
-  result = nil
-  var res = "\""
-  for i in countup(0, len(s) - 1):
-    if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, tnl)
-      app(result, toRope(res)) # reset:
-      setlen(res, 1)
-      res[0] = '\"'
-    add(res, toCChar(s[i]))
-  add(res, '\"')
-  app(result, toRope(res))
-
 proc makeLLVMString*(s: string): PRope = 
   const MaxLineLength = 64
   result = nil

compiler/cgen.nim

@@ -271,6 +271,8 @@ proc genCLineDir(r: var PRope, info: TLineInfo) =
 
 proc genLineDir(p: BProc, t: PNode) = 
   var line = t.info.safeLineNm
+  if optEmbedOrigSrc in gGlobalOptions:
+    app(p.s(cpsStmts), con(~"//", t.info.sourceLine, rnl))
   genCLineDir(p.s(cpsStmts), t.info.toFullPath, line)
   if ({optStackTrace, optEndb} * p.Options == {optStackTrace, optEndb}) and
       (p.prc == nil or sfPure notin p.prc.flags): 
@@ -278,8 +280,9 @@ proc genLineDir(p: BProc, t: PNode) =
   elif ({optLineTrace, optStackTrace} * p.Options ==
       {optLineTrace, optStackTrace}) and
       (p.prc == nil or sfPure notin p.prc.flags):
-    lineF(p, cpsStmts, "F.line = $1;F.filename = $2;$n", 
+   
-        [toRope(line), makeCString(toFilename(t.info).extractFilename)])
+    linefmt(p, cpsStmts, "nimln($1, $2);$n",
+            line.toRope, t.info.quotedFilename)
 
 include "ccgtypes.nim"
 
@@ -696,33 +699,17 @@ proc generateHeaders(m: BModule) =
       appf(m.s[cfsHeaders], "$N#include $1$N", [toRope(it.data)])
     it = PStrEntry(it.Next)
 
-proc getFrameDecl(p: BProc) = 
-  var slots: PRope
-  if p.frameLen > 0: 
-    discard cgsym(p.module, "TVarSlot")
-    slots = ropeff("  TVarSlot s[$1];$n", ", [$1 x %TVarSlot]", 
-                   [toRope(p.frameLen)])
-  else: 
-    slots = nil
-  lineFF(p, cpsLocals, "volatile struct {TFrame* prev;" &
-      "NCSTRING procname;NI line;NCSTRING filename;" &
-      "NI len;$1} F;$n",
-      "%TF = type {%TFrame*, i8*, %NI, %NI$1}$n" & 
-      "%F = alloca %TF$n", [slots])
-  inc(p.labels)
-  prepend(p.s(cpsInit), indentLine(p, ropeff("F.len = $1;$n", 
-      "%LOC$2 = getelementptr %TF %F, %NI 4$n" &
-      "store %NI $1, %NI* %LOC$2$n", [toRope(p.frameLen), toRope(p.labels)])))
-
 proc retIsNotVoid(s: PSym): bool = 
   result = (s.typ.sons[0] != nil) and not isInvalidReturnType(s.typ.sons[0])
 
 proc initFrame(p: BProc, procname, filename: PRope): PRope =
-  result = ropecg(p.module, 
+  discard cgsym(p.module, "pushFrame")
-    "\tF.procname = $1;$n" &
+  if p.frameLen > 0:
-    "\tF.filename = $2;$n" & 
+    discard cgsym(p.module, "TVarSlot")
-    "\tF.line = 0;$n" & 
+    result = rfmt(nil, "\tnimfrs($1, $2, $3)$N",
-    "\t#pushFrame((TFrame*)&F);$n", [procname, filename])
+                  procname, filename, p.frameLen.toRope)
+  else:
+    result = rfmt(nil, "\tnimfr($1, $2)$N", procname, filename)
 
 proc deinitFrame(p: BProc): PRope =
   result = rfmt(p.module, "\t#popFrame();$n")
@@ -774,11 +761,9 @@ proc genProcAux(m: BModule, prc: PSym) =
     generatedProc = rfmt(nil, "$N$1 {$N", header)
     app(generatedProc, initGCFrame(p))
     if optStackTrace in prc.options: 
-      getFrameDecl(p)
       app(generatedProc, p.s(cpsLocals))
       var procname = CStringLit(p, generatedProc, prc.name.s)
-      var filename = CStringLit(p, generatedProc, toFilename(prc.info))
+      app(generatedProc, initFrame(p, procname, prc.info.quotedFilename))
-      app(generatedProc, initFrame(p, procname, filename))
     else: 
       app(generatedProc, p.s(cpsLocals))
     if (optProfiler in prc.options) and (gCmd != cmdCompileToLLVM):
@@ -1014,11 +999,6 @@ proc genInitCode(m: BModule) =
   if m.nimTypes > 0: 
     appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", 
           [m.nimTypesName, toRope(m.nimTypes)])
-  if optStackTrace in m.initProc.options and not m.FrameDeclared:
-    # BUT: the generated init code might depend on a current frame, so
-    # declare it nevertheless:
-    m.FrameDeclared = true
-    getFrameDecl(m.initProc)
   
   app(prc, initGCFrame(m.initProc))
  
@@ -1027,10 +1007,15 @@ proc genInitCode(m: BModule) =
   app(prc, m.preInitProc.s(cpsLocals))
   app(prc, genSectionEnd(cpsLocals))
 
-  if optStackTrace in m.initProc.options and not m.PreventStackTrace: 
+  if optStackTrace in m.initProc.options and not m.FrameDeclared:
+    # BUT: the generated init code might depend on a current frame, so
+    # declare it nevertheless:
+    m.FrameDeclared = true
+    if not m.PreventStackTrace:
       var procname = CStringLit(m.initProc, prc, m.module.name.s)
-    var filename = CStringLit(m.initProc, prc, toFilename(m.module.info))
+      app(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
-    app(prc, initFrame(m.initProc, procname, filename))
+    else:
+      app(prc, ~"\tvolatile TFrame F; F.len = 0;$N")
     
   app(prc, genSectionStart(cpsInit))
   app(prc, m.preInitProc.s(cpsInit))

compiler/commands.nim

@@ -257,6 +257,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "debuginfo": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optCDebug)
+  of "embedsrc":
+    expectNoArg(switch, arg, pass, info)
+    incl(gGlobalOptions, optEmbedOrigSrc)
   of "compileonly", "c": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optCompileOnly)

compiler/lexer.nim

@@ -509,15 +509,30 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     else: lexMessage(L, errInvalidCharacterConstant)
   else: lexMessage(L, errInvalidCharacterConstant)
 
+proc newString(s: cstring, l: int): string =
+  ## XXX, how come there is no support for this?
+  result = newString(l)
+  for i in 0 .. <l:
+    result[i] = s[i]
+
 proc HandleCRLF(L: var TLexer, pos: int): int =
+  template registerLine =
+    let col = L.getColNumber(pos)
+    
+    if col > MaxLineLength:
+      lexMessagePos(L, hintLineTooLong, pos)
+
+    if optEmbedOrigSrc in gGlobalOptions:
+      let lineStart = cast[TAddress](L.buf) + L.lineStart
+      let line = newString(cast[cstring](lineStart), col)
+      addSourceLine(L.fileIdx, line)
+  
   case L.buf[pos]
   of CR:
-    if getColNumber(L, pos) > MaxLineLength: 
+    registerLine()
-      lexMessagePos(L, hintLineTooLong, pos)
     result = lexbase.HandleCR(L, pos)
   of LF:
-    if getColNumber(L, pos) > MaxLineLength: 
+    registerLine()
-      lexMessagePos(L, hintLineTooLong, pos)
     result = lexbase.HandleLF(L, pos)
   else: result = pos
   

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, sockets
+  options, strutils, os, tables, sockets, ropes, platform
 
 type 
   TMsgKind* = enum 
@@ -404,6 +404,14 @@ type
     fullPath*: string          # This is a canonical full filesystem path
     projPath*: string          # This is relative to the project's root
     
+    quotedName*: PRope         # cached quoted short name for codegen
+                               # purpoes
+    
+    lines*: seq[PRope]         # the source code of the module
+                               #   used for better error messages and
+                               #   embedding the original source in the
+                               #   generated code
+
   TLineInfo*{.final.} = object # This is designed to be as small as possible,
                                # because it is used
                                # in syntax nodes. We safe space here by using 
@@ -424,11 +432,40 @@ var
   fileInfos*: seq[TFileInfo] = @[]
   SystemFileIdx*: int32
 
+proc toCChar*(c: Char): string = 
+  case c
+  of '\0'..'\x1F', '\x80'..'\xFF': result = '\\' & toOctal(c)
+  of '\'', '\"', '\\': result = '\\' & c
+  else: result = $(c)
+
+proc makeCString*(s: string): PRope =
+  # BUGFIX: We have to split long strings into many ropes. Otherwise
+  # this could trigger an InternalError(). See the ropes module for
+  # further information.
+  const 
+    MaxLineLength = 64
+  result = nil
+  var res = "\""
+  for i in countup(0, len(s) - 1):
+    if (i + 1) mod MaxLineLength == 0:
+      add(res, '\"')
+      add(res, tnl)
+      app(result, toRope(res)) # reset:
+      setlen(res, 1)
+      res[0] = '\"'
+    add(res, toCChar(s[i]))
+  add(res, '\"')
+  app(result, toRope(res))
+
+
 proc newFileInfo(fullPath, projPath: string): TFileInfo =
   result.fullPath = fullPath
   #shallow(result.fullPath)
   result.projPath = projPath
   #shallow(result.projPath)
+  result.quotedName = projPath.extractFilename.makeCString
+  if optEmbedOrigSrc in gGlobalOptions or true:
+    result.lines = @[]
 
 proc fileInfoIdx*(filename: string): int32 =
   var
@@ -524,11 +561,11 @@ proc getInfoContext*(index: int): TLineInfo =
   if i >=% L: result = UnknownLineInfo()
   else: result = msgContext[i]
 
-proc ToFilename*(info: TLineInfo): string =
+proc toFilename*(info: TLineInfo): string =
   if info.fileIndex < 0: result = "???"
   else: result = fileInfos[info.fileIndex].projPath
 
-proc ToFilename*(fileIdx: int32): string =
+proc toFilename*(fileIdx: int32): string =
   if fileIdx < 0: result = "???"
   else: result = fileInfos[fileIdx].projPath
 
@@ -536,7 +573,7 @@ proc toFullPath*(info: TLineInfo): string =
   if info.fileIndex < 0: result = "???"
   else: result = fileInfos[info.fileIndex].fullPath
   
-proc ToLinenumber*(info: TLineInfo): int {.inline.} = 
+proc toLinenumber*(info: TLineInfo): int {.inline.} = 
   result = info.line
 
 proc toColumn*(info: TLineInfo): int {.inline.} = 
@@ -714,3 +751,28 @@ template AssertNotNil*(e: expr): expr =
 
 template InternalAssert*(e: bool): stmt =
   if not e: InternalError($InstantiationInfo())
+
+proc addSourceLine*(fileIdx: int32, line: string) =
+  fileInfos[fileIdx].lines.add line.toRope
+
+proc sourceLine*(i: TLineInfo): PRope =
+  if i.fileIndex < 0: return nil
+  InternalAssert i.fileIndex < fileInfos.len and
+                 i.line <= fileInfos[i.fileIndex].lines.len
+
+  result = fileInfos[i.fileIndex].lines[i.line-1]
+
+proc quotedFilename*(i: TLineInfo): PRope =
+  InternalAssert i.fileIndex >= 0
+  result = fileInfos[i.fileIndex].quotedName
+
+ropes.ErrorHandler = proc (err: TRopesError, msg: string, useWarning: bool) =
+  case err
+  of rInvalidFormatStr:
+    internalError("ropes: invalid format string: " & msg)
+  of rTokenTooLong:
+    internalError("ropes: token too long: " & msg)
+  of rCannotOpenFile:
+    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
+               msg)
+ 

compiler/options.nim

@@ -30,6 +30,7 @@ type                          # please make sure we have under 32 options
     optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
     optPatterns               # en/disable pattern matching
+
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake, optBoehmGC, optRefcGC, optDeadCodeElim, 
@@ -59,6 +60,7 @@ type                          # please make sure we have under 32 options
     optTaintMode,             # taint mode turned on
     optTlsEmulation,          # thread var emulation turned on
     optGenIndex               # generate index file for documentation;
+    optEmbedOrigSrc           # embed the original source in the generated code
                               # also: generate header file
    
   TGlobalOptions* = set[TGlobalOption]
@@ -236,4 +238,5 @@ proc binaryStrSearch*(x: openarray[string], y: string): int =
 template nimdbg*: expr = c.module.fileIdx == gProjectMainIdx
 template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx
 template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx
+template lnimdbg*: expr = L.fileIdx == gProjectMainIdx
 

compiler/ropes.nim

@@ -56,7 +56,7 @@
 #  To cache them they are inserted in a `cache` array.
 
 import 
-  msgs, strutils, platform, hashes, crc, options
+  strutils, platform, hashes, crc, options
 
 type
   TFormatStr* = string # later we may change it to CString for better
@@ -72,6 +72,11 @@ type
   
   TRopeSeq* = seq[PRope]
 
+  TRopesError* = enum
+    rCannotOpenFile
+    rInvalidFormatStr
+    rTokenTooLong
+
 proc con*(a, b: PRope): PRope
 proc con*(a: PRope, b: string): PRope
 proc con*(a: string, b: PRope): PRope
@@ -92,6 +97,9 @@ proc RopeInvariant*(r: PRope): bool
   # exported for debugging
 # implementation
 
+var ErrorHandler*: proc(err: TRopesError, msg: string, useWarning = false)
+  # avoid dependency on msgs.nim
+  
 proc ropeLen(a: PRope): int = 
   if a == nil: result = 0
   else: result = a.length
@@ -225,8 +233,7 @@ proc WriteRope*(head: PRope, filename: string, useWarning = false) =
     if head != nil: WriteRope(f, head)
     close(f)
   else:
-    rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile,
+    ErrorHandler(rCannotOpenFile, filename, useWarning)
-               filename)
 
 var
   rnl* = tnl.newRope
@@ -256,7 +263,7 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
           if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break 
         num = j
         if j > high(args) + 1:
-          internalError("ropes: invalid format string $" & $(j))
+          ErrorHandler(rInvalidFormatStr, $(j))
         else:
           app(result, args[j - 1])
       of 'n':
@@ -265,7 +272,8 @@ proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope =
       of 'N':
         app(result, rnl)
         inc(i)
-      else: InternalError("ropes: invalid format string $" & frmt[i])
+      else:
+        ErrorHandler(rInvalidFormatStr, $(frmt[i]))
     var start = i
     while i < length:
       if frmt[i] != '$': inc(i)
@@ -291,7 +299,7 @@ const
 proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = 
   if r.data != nil:
     if r.length > bufSize:
-      internalError("ropes: token too long")
+      ErrorHandler(rTokenTooLong, r.data)
       return
     var readBytes = readBuffer(bin, buf, r.length)
     result = readBytes == r.length and

doc/advopt.txt

@@ -52,6 +52,8 @@ Advanced options:
                             (Nimrod, mangled) identifier pairs
   --project                 document the whole project (doc2)
   --lineDir:on|off          generation of #line directive on|off
+  --embedsrc                embeds the original source code as comments
+                            in the generated output
   --threadanalysis:on|off   turn thread analysis on|off
   --tlsEmulation:on|off     turn thread local storage emulation on|off
   --taintMode:on|off        turn taint mode on|off

lib/nimbase.h

@@ -436,6 +436,17 @@ struct TFrame {
   NI len;
 };
 
+#define nimfr(proc, file) \
+  volatile TFrame F; \
+  F.procname = proc; F.filename = file; F.line = 0; F.len = 0; pushFrame(&F);
+
+#define nimfrs(proc, file, slots) \
+  volatile struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; TVarSlot s[slots];} F; \
+  F.procname = proc; F.filename = file; F.line = 0; F.len = slots; pushFrame((TFrame*)&F);
+
+#define nimln(n, file) \
+  F.line = n; F.filename = file;
+
 #define NIM_POSIX_INIT  __attribute__((constructor)) 
 
 #if defined(_MSCVER) && defined(__i386__)