sourcemaps for the JS codegen (#7508)
This commit is contained in:
Alexander Ivanov
GitHub
parent
2b3b24a804
commit
920add5880
4 changed files with 424 additions and 9 deletions
|
|
@ -890,6 +890,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
|
|||
processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
|
||||
if optPanics in conf.globalOptions:
|
||||
defineSymbol(conf.symbols, "nimPanics")
|
||||
of "sourcemap":
|
||||
conf.globalOptions.incl optSourcemap
|
||||
conf.options.incl optLineDir
|
||||
# processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info)
|
||||
of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
|
||||
handleStdinInput(conf)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ import
|
|||
nversion, msgs, idents, types, tables,
|
||||
ropes, math, passes, ccgutils, wordrecg, renderer,
|
||||
intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
|
||||
transf, injectdestructors
|
||||
transf, injectdestructors, sourcemap, json, sets
|
||||
|
||||
|
||||
from modulegraphs import ModuleGraph, PPassContext
|
||||
|
|
@ -205,6 +205,8 @@ proc mapType(typ: PType): TJSTypeKind =
|
|||
proc mapType(p: PProc; typ: PType): TJSTypeKind =
|
||||
result = mapType(typ)
|
||||
|
||||
var mangled = initSet[string]()
|
||||
|
||||
proc mangleName(m: BModule, s: PSym): Rope =
|
||||
proc validJsName(name: string): bool =
|
||||
result = true
|
||||
|
|
@ -259,6 +261,11 @@ proc mangleName(m: BModule, s: PSym): Rope =
|
|||
result.add(rope(s.id))
|
||||
s.loc.r = result
|
||||
|
||||
# TODO: optimize
|
||||
let s = $result
|
||||
if '_' in s:
|
||||
mangled.incl(s)
|
||||
|
||||
proc escapeJSString(s: string): string =
|
||||
result = newStringOfCap(s.len + s.len shr 2)
|
||||
result.add("\"")
|
||||
|
|
@ -641,11 +648,16 @@ proc hasFrameInfo(p: PProc): bool =
|
|||
({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
|
||||
((p.prc == nil) or not (sfPure in p.prc.flags))
|
||||
|
||||
proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope =
|
||||
ropes.`%`("// line $2 \"$1\"$n",
|
||||
[rope(toFullPath(config, info)), rope(line)])
|
||||
|
||||
proc genLineDir(p: PProc, n: PNode) =
|
||||
let line = toLinenumber(n.info)
|
||||
if optLineDir in p.options:
|
||||
lineF(p, "// line $2 \"$1\"$n",
|
||||
[rope(toFilename(p.config, n.info)), rope(line)])
|
||||
if line < 0:
|
||||
return
|
||||
if optLineDir in p.options or optLineDir in p.config.options:
|
||||
lineF(p, "$1", [lineDir(p.config, n.info, line)])
|
||||
if hasFrameInfo(p):
|
||||
lineF(p, "F.line = $1;$n", [rope(line)])
|
||||
|
||||
|
|
@ -2241,6 +2253,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
|
|||
|
||||
p.nested: genStmt(p, transformedBody)
|
||||
|
||||
|
||||
if optLineDir in p.config.options:
|
||||
result = lineDir(p.config, prc.info, toLinenumber(prc.info))
|
||||
|
||||
var def: Rope
|
||||
if not prc.constraint.isNil:
|
||||
def = runtimeFormat(prc.constraint.strVal & " {$n$#$#$#$#$#",
|
||||
|
|
@ -2253,7 +2269,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
|
|||
optionalLine(genProcBody(p, prc)),
|
||||
optionalLine(p.indentLine(returnStmt))])
|
||||
else:
|
||||
result = ~"\L"
|
||||
# if optLineDir in p.config.options:
|
||||
# result.add(~"\L")
|
||||
|
||||
if p.config.hcrOn:
|
||||
# Here, we introduce thunks that create the equivalent of a jump table
|
||||
|
|
@ -2336,6 +2353,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
|
|||
if r.kind != resCallee: r.kind = resNone
|
||||
#r.address = nil
|
||||
r.res = nil
|
||||
|
||||
case n.kind
|
||||
of nkSym:
|
||||
genSym(p, n, r)
|
||||
|
|
@ -2382,14 +2400,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
|
|||
else: r.res = rope(f.toStrMaxPrecision)
|
||||
r.kind = resExpr
|
||||
of nkCallKinds:
|
||||
if isEmptyType(n.typ): genLineDir(p, n)
|
||||
if isEmptyType(n.typ):
|
||||
genLineDir(p, n)
|
||||
if (n[0].kind == nkSym) and (n[0].sym.magic != mNone):
|
||||
genMagic(p, n, r)
|
||||
elif n[0].kind == nkSym and sfInfixCall in n[0].sym.flags and
|
||||
n.len >= 1:
|
||||
genInfixCall(p, n, r)
|
||||
else:
|
||||
genCall(p, n, r)
|
||||
genCall(p, n, r)
|
||||
of nkClosure: gen(p, n[0], r)
|
||||
of nkCurly: genSetConstr(p, n, r)
|
||||
of nkBracket: genArrayConstr(p, n, r)
|
||||
|
|
@ -2585,10 +2604,15 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
|
|||
n.add destructorCall
|
||||
if passes.skipCodegen(m.config, n): return n
|
||||
if sfMainModule in m.module.flags:
|
||||
let code = wholeCode(graph, m)
|
||||
var code = genHeader() & wholeCode(graph, m)
|
||||
let outFile = m.config.prepareToWriteOutput()
|
||||
discard writeRopeIfNotEqual(genHeader() & code, outFile)
|
||||
|
||||
if optSourcemap in m.config.globalOptions:
|
||||
var map: SourceMap
|
||||
(code, map) = genSourceMap($(code), mangled, outFile.string)
|
||||
writeFile(outFile.string & ".map", $(%map))
|
||||
discard writeRopeIfNotEqual(code, outFile)
|
||||
|
||||
proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
|
||||
result = newModule(graph, s)
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ type # please make sure we have under 32 options
|
|||
optOldAst,
|
||||
optSinkInference # 'sink T' inference
|
||||
|
||||
|
||||
TOptions* = set[TOption]
|
||||
TGlobalOption* = enum # **keep binary compatible**
|
||||
gloptNone, optForceFullMake,
|
||||
|
|
@ -94,6 +95,7 @@ type # please make sure we have under 32 options
|
|||
optProduceAsm # produce assembler code
|
||||
optPanics # turn panics (sysFatal) into a process termination
|
||||
optNimV1Emulation # emulate Nim v1.0
|
||||
optSourcemap
|
||||
|
||||
TGlobalOptions* = set[TGlobalOption]
|
||||
|
||||
|
|
|
|||
385
compiler/sourcemap.nim
Normal file
385
compiler/sourcemap.nim
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
import os, strformat, strutils, tables, sets, ropes, json, algorithm
|
||||
|
||||
type
|
||||
SourceNode* = ref object
|
||||
line*: int
|
||||
column*: int
|
||||
source*: string
|
||||
name*: string
|
||||
children*: seq[Child]
|
||||
|
||||
C = enum cSourceNode, cString
|
||||
|
||||
Child* = ref object
|
||||
case kind*: C:
|
||||
of cSourceNode:
|
||||
node*: SourceNode
|
||||
of cString:
|
||||
s*: string
|
||||
|
||||
SourceMap* = ref object
|
||||
version*: int
|
||||
sources*: seq[string]
|
||||
names*: seq[string]
|
||||
mappings*: string
|
||||
file*: string
|
||||
# sourceRoot*: string
|
||||
# sourcesContent*: string
|
||||
|
||||
SourceMapGenerator = ref object
|
||||
file: string
|
||||
sourceRoot: string
|
||||
skipValidation: bool
|
||||
sources: seq[string]
|
||||
names: seq[string]
|
||||
mappings: seq[Mapping]
|
||||
|
||||
Mapping* = ref object
|
||||
source*: string
|
||||
original*: tuple[line: int, column: int]
|
||||
generated*: tuple[line: int, column: int]
|
||||
name*: string
|
||||
noSource*: bool
|
||||
noName*: bool
|
||||
|
||||
|
||||
proc child*(s: string): Child =
|
||||
Child(kind: cString, s: s)
|
||||
|
||||
|
||||
proc child*(node: SourceNode): Child =
|
||||
Child(kind: cSourceNode, node: node)
|
||||
|
||||
|
||||
proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode =
|
||||
SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)])
|
||||
|
||||
|
||||
proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode =
|
||||
SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)])
|
||||
|
||||
|
||||
proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode =
|
||||
SourceNode(line: line, column: column, source: path, name: name, children: children)
|
||||
|
||||
|
||||
|
||||
|
||||
# debugging
|
||||
|
||||
|
||||
proc text*(sourceNode: SourceNode, depth: int): string =
|
||||
let empty = " "
|
||||
result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n"
|
||||
for child in sourceNode.children:
|
||||
if child.kind == cString:
|
||||
result.add(&"{repeat(empty, depth + 1)}{child.s}\n")
|
||||
else:
|
||||
result.add(child.node.text(depth + 1))
|
||||
|
||||
|
||||
proc `$`*(sourceNode: SourceNode): string =
|
||||
text(sourceNode, 0)
|
||||
|
||||
|
||||
# base64_VLQ
|
||||
|
||||
|
||||
let integers = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
|
||||
|
||||
|
||||
proc encode*(i: int): string =
|
||||
result = ""
|
||||
var n = i
|
||||
if n < 0:
|
||||
n = (-n shl 1) or 1
|
||||
else:
|
||||
n = n shl 1
|
||||
|
||||
var z = 0
|
||||
while z == 0 or n > 0:
|
||||
var e = n and 31
|
||||
n = n shr 5
|
||||
if n > 0:
|
||||
e = e or 32
|
||||
|
||||
result.add(integers[e])
|
||||
z += 1
|
||||
|
||||
|
||||
type
|
||||
TokenState = enum Normal, String, Ident, Mangled
|
||||
|
||||
iterator tokenize*(line: string): (bool, string) =
|
||||
# result = @[]
|
||||
var state = Normal
|
||||
var token = ""
|
||||
var isMangled = false
|
||||
for z, ch in line:
|
||||
if ch.isAlphaAscii:
|
||||
if state == Normal:
|
||||
state = Ident
|
||||
if token.len > 0:
|
||||
yield (isMangled, token)
|
||||
token = $ch
|
||||
isMangled = false
|
||||
else:
|
||||
token.add(ch)
|
||||
elif ch == '_':
|
||||
if state == Ident:
|
||||
state = Mangled
|
||||
isMangled = true
|
||||
token.add($ch)
|
||||
elif ch != '"' and not ch.isAlphaNumeric:
|
||||
if state in {Ident, Mangled}:
|
||||
state = Normal
|
||||
if token.len > 0:
|
||||
yield (isMangled, token)
|
||||
token = $ch
|
||||
isMangled = false
|
||||
else:
|
||||
token.add($ch)
|
||||
elif ch == '"':
|
||||
if state != String:
|
||||
state = String
|
||||
if token.len > 0:
|
||||
yield (isMangled, token)
|
||||
token = $ch
|
||||
isMangled = false
|
||||
else:
|
||||
state = Normal
|
||||
token.add($ch)
|
||||
if token.len > 0:
|
||||
yield (isMangled, token)
|
||||
isMangled = false
|
||||
token = ""
|
||||
else:
|
||||
token.add($ch)
|
||||
if token.len > 0:
|
||||
yield (isMangled, token)
|
||||
|
||||
proc parse*(source: string, path: string, mangled: HashSet[string]): SourceNode =
|
||||
let lines = source.splitLines()
|
||||
var lastLocation: SourceNode = nil
|
||||
result = newSourceNode(0, 0, path, @[])
|
||||
|
||||
# we just use one single parent and add all nim lines
|
||||
# as its children, I guess in typical codegen
|
||||
# that happens recursively on ast level
|
||||
# we also don't have column info, but I doubt more one nim lines can compile to one js
|
||||
# maybe in macros?
|
||||
|
||||
for i, originalLine in lines:
|
||||
let line = originalLine.strip
|
||||
if line.len == 0:
|
||||
continue
|
||||
|
||||
# this shouldn't be a problem:
|
||||
# jsgen doesn't generate comments
|
||||
# and if you emit // line you probably know what you're doing
|
||||
if line.startsWith("// line"):
|
||||
if result.children.len > 0:
|
||||
result.children[^1].node.children.add(child(line & "\n"))
|
||||
let pos = line.find(" ", 8)
|
||||
let lineNumber = line[8 .. pos - 1].parseInt
|
||||
let linePath = line[pos + 2 .. ^2] # quotes
|
||||
|
||||
lastLocation = newSourceNode(
|
||||
lineNumber,
|
||||
0,
|
||||
linePath,
|
||||
@[])
|
||||
result.children.add(child(lastLocation))
|
||||
else:
|
||||
var last: SourceNode
|
||||
for token in line.tokenize():
|
||||
var name = ""
|
||||
if token[0]:
|
||||
name = token[1].split('_', 1)[0]
|
||||
|
||||
|
||||
if result.children.len > 0:
|
||||
result.children[^1].node.children.add(
|
||||
child(
|
||||
newSourceNode(
|
||||
result.children[^1].node.line,
|
||||
0,
|
||||
result.children[^1].node.source,
|
||||
token[1],
|
||||
name)))
|
||||
last = result.children[^1].node.children[^1].node
|
||||
else:
|
||||
result.children.add(
|
||||
child(
|
||||
newSourceNode(i + 1, 0, path, token[1], name)))
|
||||
last = result.children[^1].node
|
||||
let nl = "\n"
|
||||
if not last.isNil:
|
||||
last.source.add(nl)
|
||||
|
||||
proc cmp(a: Mapping, b: Mapping): int =
|
||||
var c = cmp(a.generated, b.generated)
|
||||
if c != 0:
|
||||
return c
|
||||
|
||||
c = cmp(a.source, b.source)
|
||||
if c != 0:
|
||||
return c
|
||||
|
||||
c = cmp(a.original, b.original)
|
||||
if c != 0:
|
||||
return c
|
||||
|
||||
return cmp(a.name, b.name)
|
||||
|
||||
|
||||
proc index*[T](elements: seq[T], element: T): int =
|
||||
for z in 0 ..< elements.len:
|
||||
if elements[z] == element:
|
||||
return z
|
||||
return -1
|
||||
|
||||
|
||||
proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string =
|
||||
var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "")
|
||||
var previousSourceId = 0
|
||||
var previousNameId = 0
|
||||
result = ""
|
||||
var next = ""
|
||||
var nameId = 0
|
||||
var sourceId = 0
|
||||
|
||||
for z, mapping in mappings:
|
||||
next = ""
|
||||
|
||||
if mapping.generated.line != previous.generated.line:
|
||||
previous.generated.column = 0
|
||||
|
||||
while mapping.generated.line != previous.generated.line:
|
||||
next.add(";")
|
||||
previous.generated.line += 1
|
||||
|
||||
else:
|
||||
if z > 0:
|
||||
if cmp(mapping, mappings[z - 1]) == 0:
|
||||
continue
|
||||
next.add(",")
|
||||
|
||||
next.add(encode(mapping.generated.column - previous.generated.column))
|
||||
previous.generated.column = mapping.generated.column
|
||||
|
||||
if not mapping.noSource and mapping.source.len > 0:
|
||||
sourceId = map.sources.index(mapping.source)
|
||||
next.add(encode(sourceId - previousSourceId))
|
||||
previousSourceId = sourceId
|
||||
next.add(encode(mapping.original.line - 1 - previous.original.line))
|
||||
previous.original.line = mapping.original.line - 1
|
||||
next.add(encode(mapping.original.column - previous.original.column))
|
||||
previous.original.column = mapping.original.column
|
||||
|
||||
if not mapping.noName and mapping.name.len > 0:
|
||||
nameId = map.names.index(mapping.name)
|
||||
next.add(encode(nameId - previousNameId))
|
||||
previousNameId = nameId
|
||||
|
||||
result.add(next)
|
||||
|
||||
|
||||
proc gen*(map: SourceMapGenerator): SourceMap =
|
||||
var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int:
|
||||
cmp(a, b)
|
||||
result = SourceMap(
|
||||
file: map.file,
|
||||
version: 3,
|
||||
sources: map.sources[0..^1],
|
||||
names: map.names[0..^1],
|
||||
mappings: map.serializeMappings(mappings))
|
||||
|
||||
|
||||
|
||||
proc addMapping*(map: SourceMapGenerator, mapping: Mapping) =
|
||||
if not mapping.noSource and mapping.source notin map.sources:
|
||||
map.sources.add(mapping.source)
|
||||
|
||||
if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names:
|
||||
map.names.add(mapping.name)
|
||||
|
||||
# echo "map ", mapping.source, " ", mapping.original, " ", mapping.generated, " ", mapping.name
|
||||
map.mappings.add(mapping)
|
||||
|
||||
|
||||
proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) =
|
||||
for child in node.children:
|
||||
if child.kind == cString and child.s.len > 0:
|
||||
fn(child.s, node)
|
||||
else:
|
||||
child.node.walk(fn)
|
||||
|
||||
|
||||
proc toSourceMap*(node: SourceNode, file: string): SourceMapGenerator =
|
||||
var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[])
|
||||
|
||||
var generated = (line: 1, column: 0)
|
||||
var sourceMappingActive = false
|
||||
var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[])
|
||||
|
||||
node.walk do (line: string, original: SourceNode):
|
||||
if original.source.endsWith(".js"):
|
||||
# ignore it
|
||||
discard
|
||||
else:
|
||||
if original.line != -1:
|
||||
if lastOriginal.source != original.source or
|
||||
lastOriginal.line != original.line or
|
||||
lastOriginal.column != original.column or
|
||||
lastOriginal.name != original.name:
|
||||
map.addMapping(
|
||||
Mapping(
|
||||
source: original.source,
|
||||
original: (line: original.line, column: original.column),
|
||||
generated: (line: generated.line, column: generated.column),
|
||||
name: original.name))
|
||||
|
||||
lastOriginal = SourceNode(
|
||||
source: original.source,
|
||||
line: original.line,
|
||||
column: original.column,
|
||||
name: original.name,
|
||||
children: lastOriginal.children)
|
||||
sourceMappingActive = true
|
||||
elif sourceMappingActive:
|
||||
map.addMapping(
|
||||
Mapping(
|
||||
noSource: true,
|
||||
noName: true,
|
||||
generated: (line: generated.line, column: generated.column),
|
||||
original: (line: -1, column: -1)))
|
||||
lastOriginal.line = -1
|
||||
sourceMappingActive = false
|
||||
|
||||
for z in 0 ..< line.len:
|
||||
if line[z] in Newlines:
|
||||
generated.line += 1
|
||||
generated.column = 0
|
||||
|
||||
if z == line.len - 1:
|
||||
lastOriginal.line = -1
|
||||
sourceMappingActive = false
|
||||
elif sourceMappingActive:
|
||||
map.addMapping(
|
||||
Mapping(
|
||||
source: original.source,
|
||||
original: (line: original.line, column: original.column),
|
||||
generated: (line: generated.line, column: generated.column),
|
||||
name: original.name))
|
||||
else:
|
||||
generated.column += 1
|
||||
|
||||
map
|
||||
|
||||
|
||||
proc genSourceMap*(source: string, mangled: HashSet[string], outFile: string): (Rope, SourceMap) =
|
||||
let node = parse(source, outFile, mangled)
|
||||
let map = node.toSourceMap(file = outFile)
|
||||
((&"{source}\n//# sourceMappingURL={outFile}.map").rope, map.gen)
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue