sourcemaps for the JS codegen (#7508)

compiler/commands.nim

@@ -890,6 +890,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
     if optPanics in conf.globalOptions:
       defineSymbol(conf.symbols, "nimPanics")
+  of "sourcemap":
+    conf.globalOptions.incl optSourcemap
+    conf.options.incl optLineDir
+    # processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
   else:

compiler/jsgen.nim

@@ -33,7 +33,7 @@ import
   nversion, msgs, idents, types, tables,
   ropes, math, passes, ccgutils, wordrecg, renderer,
   intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
-  transf, injectdestructors
+  transf, injectdestructors, sourcemap, json, sets
 
 
 from modulegraphs import ModuleGraph, PPassContext
@@ -205,6 +205,8 @@ proc mapType(typ: PType): TJSTypeKind =
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   result = mapType(typ)
 
+var mangled = initSet[string]()
+
 proc mangleName(m: BModule, s: PSym): Rope =
   proc validJsName(name: string): bool =
     result = true
@@ -259,6 +261,11 @@ proc mangleName(m: BModule, s: PSym): Rope =
         result.add(rope(s.id))
     s.loc.r = result
 
+  # TODO: optimize
+  let s = $result
+  if '_' in s:
+    mangled.incl(s)
+
 proc escapeJSString(s: string): string =
   result = newStringOfCap(s.len + s.len shr 2)
   result.add("\"")
@@ -641,11 +648,16 @@ proc hasFrameInfo(p: PProc): bool =
   ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
       ((p.prc == nil) or not (sfPure in p.prc.flags))
 
+proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope =
+  ropes.`%`("// line $2 \"$1\"$n",
+         [rope(toFullPath(config, info)), rope(line)])
+
 proc genLineDir(p: PProc, n: PNode) =
   let line = toLinenumber(n.info)
-  if optLineDir in p.options:
+  if line < 0:
-    lineF(p, "// line $2 \"$1\"$n",
+    return
-         [rope(toFilename(p.config, n.info)), rope(line)])
+  if optLineDir in p.options or optLineDir in p.config.options:
+    lineF(p, "$1", [lineDir(p.config, n.info, line)])
   if hasFrameInfo(p):
     lineF(p, "F.line = $1;$n", [rope(line)])
 
@@ -2241,6 +2253,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
 
   p.nested: genStmt(p, transformedBody)
 
+
+  if optLineDir in p.config.options:
+    result = lineDir(p.config, prc.info, toLinenumber(prc.info))
+
   var def: Rope
   if not prc.constraint.isNil:
     def = runtimeFormat(prc.constraint.strVal & " {$n$#$#$#$#$#",
@@ -2253,7 +2269,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
               optionalLine(genProcBody(p, prc)),
               optionalLine(p.indentLine(returnStmt))])
   else:
-    result = ~"\L"
+    # if optLineDir in p.config.options:
+      # result.add(~"\L")
 
     if p.config.hcrOn:
       # Here, we introduce thunks that create the equivalent of a jump table
@@ -2336,6 +2353,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   if r.kind != resCallee: r.kind = resNone
   #r.address = nil
   r.res = nil
+  
   case n.kind
   of nkSym:
     genSym(p, n, r)
@@ -2382,14 +2400,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     else: r.res = rope(f.toStrMaxPrecision)
     r.kind = resExpr
   of nkCallKinds:
-    if isEmptyType(n.typ): genLineDir(p, n)
+    if isEmptyType(n.typ):
+      genLineDir(p, n)
     if (n[0].kind == nkSym) and (n[0].sym.magic != mNone):
       genMagic(p, n, r)
     elif n[0].kind == nkSym and sfInfixCall in n[0].sym.flags and
         n.len >= 1:
       genInfixCall(p, n, r)
     else:
-      genCall(p, n, r)
+      genCall(p, n, r)    
   of nkClosure: gen(p, n[0], r)
   of nkCurly: genSetConstr(p, n, r)
   of nkBracket: genArrayConstr(p, n, r)
@@ -2585,10 +2604,15 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       n.add destructorCall
   if passes.skipCodegen(m.config, n): return n
   if sfMainModule in m.module.flags:
-    let code = wholeCode(graph, m)
+    var code = genHeader() & wholeCode(graph, m)
     let outFile = m.config.prepareToWriteOutput()
-    discard writeRopeIfNotEqual(genHeader() & code, outFile)
 
+    if optSourcemap in m.config.globalOptions:
+      var map: SourceMap
+      (code, map) = genSourceMap($(code), mangled, outFile.string)
+      writeFile(outFile.string & ".map", $(%map))
+    discard writeRopeIfNotEqual(code, outFile)
+    
 proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
   result = newModule(graph, s)
 

compiler/options.nim

@@ -44,6 +44,7 @@ type                          # please make sure we have under 32 options
     optOldAst,
     optSinkInference          # 'sink T' inference
 
+
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake,
@@ -94,6 +95,7 @@ type                          # please make sure we have under 32 options
     optProduceAsm             # produce assembler code
     optPanics                 # turn panics (sysFatal) into a process termination
     optNimV1Emulation         # emulate Nim v1.0
+    optSourcemap
 
   TGlobalOptions* = set[TGlobalOption]
 

compiler/sourcemap.nim

@@ -0,0 +1,385 @@
+import os, strformat, strutils, tables, sets, ropes, json, algorithm
+
+type
+  SourceNode* = ref object
+    line*:      int
+    column*:    int
+    source*:    string
+    name*:      string
+    children*:  seq[Child]
+
+  C = enum cSourceNode, cString
+
+  Child* = ref object
+    case kind*: C:
+    of cSourceNode:
+      node*:  SourceNode
+    of cString:
+      s*:     string
+
+  SourceMap* = ref object
+    version*:   int
+    sources*:   seq[string]
+    names*:     seq[string]
+    mappings*:  string
+    file*:      string
+    # sourceRoot*: string
+    # sourcesContent*: string
+
+  SourceMapGenerator = ref object
+    file:           string
+    sourceRoot:     string
+    skipValidation: bool
+    sources:        seq[string]
+    names:          seq[string]
+    mappings:       seq[Mapping]
+
+  Mapping* = ref object
+    source*:        string
+    original*:      tuple[line: int, column: int]
+    generated*:     tuple[line: int, column: int]
+    name*:          string
+    noSource*:      bool
+    noName*:        bool
+
+
+proc child*(s: string): Child =
+  Child(kind: cString, s: s)
+
+
+proc child*(node: SourceNode): Child =
+  Child(kind: cSourceNode, node: node)
+
+
+proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode =
+  SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)])
+
+
+proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode =
+  SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)])
+
+
+proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode =
+  SourceNode(line: line, column: column, source: path, name: name, children: children)
+
+
+
+
+# debugging
+
+
+proc text*(sourceNode: SourceNode, depth: int): string =
+  let empty = "  "
+  result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n"
+  for child in sourceNode.children:
+    if child.kind == cString:
+      result.add(&"{repeat(empty, depth + 1)}{child.s}\n")
+    else:
+      result.add(child.node.text(depth + 1))
+
+
+proc `$`*(sourceNode: SourceNode): string =
+  text(sourceNode, 0)
+
+
+# base64_VLQ
+
+
+let integers = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+
+
+proc encode*(i: int): string =
+  result = ""
+  var n = i
+  if n < 0:
+    n = (-n shl 1) or 1
+  else:
+    n = n shl 1
+
+  var z = 0
+  while z == 0 or n > 0:
+    var e = n and 31
+    n = n shr 5
+    if n > 0:
+      e = e or 32
+
+    result.add(integers[e])
+    z += 1
+
+
+type
+  TokenState = enum Normal, String, Ident, Mangled
+
+iterator tokenize*(line: string): (bool, string) =
+  # result = @[]
+  var state = Normal
+  var token = ""
+  var isMangled = false
+  for z, ch in line:
+    if ch.isAlphaAscii:
+      if state == Normal:
+        state = Ident
+        if token.len > 0:
+          yield (isMangled, token)
+        token = $ch
+        isMangled = false
+      else:
+        token.add(ch)
+    elif ch == '_':
+      if state == Ident:
+        state = Mangled
+        isMangled = true
+      token.add($ch)
+    elif ch != '"' and not ch.isAlphaNumeric:
+      if state in {Ident, Mangled}:
+        state = Normal
+        if token.len > 0:
+          yield (isMangled, token)
+        token = $ch
+        isMangled = false
+      else:
+        token.add($ch)
+    elif ch == '"':
+      if state != String:
+        state = String
+        if token.len > 0:
+          yield (isMangled, token)
+        token = $ch
+        isMangled = false
+      else:
+        state = Normal
+        token.add($ch)
+        if token.len > 0:
+          yield (isMangled, token)
+        isMangled = false
+        token = ""
+    else:
+      token.add($ch)
+  if token.len > 0:
+    yield (isMangled, token)
+
+proc parse*(source: string, path: string, mangled: HashSet[string]): SourceNode =
+  let lines = source.splitLines()
+  var lastLocation: SourceNode = nil
+  result = newSourceNode(0, 0, path, @[])
+    
+  # we just use one single parent and add all nim lines
+  # as its children, I guess in typical codegen
+  # that happens recursively on ast level
+  # we also don't have column info, but I doubt more one nim lines can compile to one js
+  # maybe in macros?
+
+  for i, originalLine in lines:
+    let line = originalLine.strip
+    if line.len == 0:
+      continue
+      
+    # this shouldn't be a problem:
+    # jsgen doesn't generate comments
+    # and if you emit // line you probably know what you're doing
+    if line.startsWith("// line"):
+      if result.children.len > 0:
+        result.children[^1].node.children.add(child(line & "\n"))
+      let pos = line.find(" ", 8)
+      let lineNumber = line[8 .. pos - 1].parseInt
+      let linePath = line[pos + 2 .. ^2] # quotes
+      
+      lastLocation = newSourceNode(
+        lineNumber,
+        0,
+        linePath,
+        @[])
+      result.children.add(child(lastLocation))
+    else:
+      var last: SourceNode
+      for token in line.tokenize():
+        var name = ""
+        if token[0]:
+          name = token[1].split('_', 1)[0]
+        
+        
+        if result.children.len > 0:
+          result.children[^1].node.children.add(
+            child(
+              newSourceNode(
+                result.children[^1].node.line,
+                0,
+                result.children[^1].node.source,
+                token[1],
+                name)))
+          last = result.children[^1].node.children[^1].node
+        else:
+          result.children.add(
+            child(
+              newSourceNode(i + 1, 0, path, token[1], name)))
+          last = result.children[^1].node
+      let nl = "\n"
+      if not last.isNil:
+        last.source.add(nl)
+
+proc cmp(a: Mapping, b: Mapping): int =
+  var c = cmp(a.generated, b.generated)
+  if c != 0:
+    return c
+
+  c = cmp(a.source, b.source)
+  if c != 0:
+    return c
+
+  c = cmp(a.original, b.original)
+  if c != 0:
+    return c
+
+  return cmp(a.name, b.name)
+
+
+proc index*[T](elements: seq[T], element: T): int =
+  for z in 0 ..< elements.len:
+    if elements[z] == element:
+      return z
+  return -1
+
+
+proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string =
+  var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "")
+  var previousSourceId = 0
+  var previousNameId = 0
+  result = ""
+  var next = ""
+  var nameId = 0
+  var sourceId = 0
+
+  for z, mapping in mappings:
+    next = ""
+
+    if mapping.generated.line != previous.generated.line:
+      previous.generated.column = 0
+
+      while mapping.generated.line != previous.generated.line:
+        next.add(";")
+        previous.generated.line += 1
+
+    else:
+      if z > 0:
+        if cmp(mapping, mappings[z - 1]) == 0:
+          continue
+        next.add(",")
+
+    next.add(encode(mapping.generated.column - previous.generated.column))
+    previous.generated.column = mapping.generated.column
+
+    if not mapping.noSource and mapping.source.len > 0:
+      sourceId = map.sources.index(mapping.source)
+      next.add(encode(sourceId - previousSourceId))
+      previousSourceId = sourceId
+      next.add(encode(mapping.original.line - 1 - previous.original.line))
+      previous.original.line = mapping.original.line - 1
+      next.add(encode(mapping.original.column - previous.original.column))
+      previous.original.column = mapping.original.column
+
+      if not mapping.noName and mapping.name.len > 0:
+        nameId = map.names.index(mapping.name)
+        next.add(encode(nameId - previousNameId))
+        previousNameId = nameId
+
+    result.add(next)
+
+
+proc gen*(map: SourceMapGenerator): SourceMap =
+  var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int:
+    cmp(a, b)
+  result = SourceMap(
+    file: map.file,
+    version: 3,
+    sources: map.sources[0..^1],
+    names: map.names[0..^1],
+    mappings: map.serializeMappings(mappings))
+
+
+
+proc addMapping*(map: SourceMapGenerator, mapping: Mapping) =
+  if not mapping.noSource and mapping.source notin map.sources:
+    map.sources.add(mapping.source)
+
+  if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names:
+    map.names.add(mapping.name)
+
+  # echo "map ", mapping.source, " ", mapping.original, " ", mapping.generated, " ", mapping.name
+  map.mappings.add(mapping)
+
+
+proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) =
+  for child in node.children:
+    if child.kind == cString and child.s.len > 0:
+      fn(child.s, node)
+    else:
+      child.node.walk(fn)
+
+
+proc toSourceMap*(node: SourceNode, file: string): SourceMapGenerator =
+  var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[])
+
+  var generated = (line: 1, column: 0)
+  var sourceMappingActive = false
+  var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[])
+
+  node.walk do (line: string, original: SourceNode):
+    if original.source.endsWith(".js"):
+      # ignore it
+      discard
+    else:
+      if original.line != -1:
+        if lastOriginal.source != original.source or
+           lastOriginal.line != original.line or
+           lastOriginal.column != original.column or
+           lastOriginal.name != original.name:
+          map.addMapping(
+            Mapping(
+              source: original.source,
+              original: (line: original.line, column: original.column),
+              generated: (line: generated.line, column: generated.column),
+              name: original.name))
+
+        lastOriginal = SourceNode(
+          source: original.source,
+          line: original.line,
+          column: original.column,
+          name: original.name,
+          children: lastOriginal.children)
+        sourceMappingActive = true
+      elif sourceMappingActive:
+        map.addMapping(
+          Mapping(
+            noSource: true,
+            noName: true,
+            generated: (line: generated.line, column: generated.column),
+            original: (line: -1, column: -1)))
+        lastOriginal.line = -1
+        sourceMappingActive = false
+
+    for z in 0 ..< line.len:
+      if line[z] in Newlines:
+        generated.line += 1
+        generated.column = 0
+
+        if z == line.len - 1:
+          lastOriginal.line = -1
+          sourceMappingActive = false
+        elif sourceMappingActive:
+          map.addMapping(
+            Mapping(
+              source: original.source,
+              original: (line: original.line, column: original.column),
+              generated: (line: generated.line, column: generated.column),
+              name: original.name))
+      else:
+        generated.column += 1
+    
+  map
+
+
+proc genSourceMap*(source: string, mangled: HashSet[string], outFile: string): (Rope, SourceMap) =
+  let node = parse(source, outFile, mangled)
+  let map = node.toSourceMap(file = outFile)
+  ((&"{source}\n//# sourceMappingURL={outFile}.map").rope, map.gen)
+