much more efficient rod file generation

This commit is contained in:
Araq 2011-10-18 17:21:51 +02:00
commit 4de84024e5
18 changed files with 611 additions and 557 deletions

View file

@ -1,7 +1,7 @@
#
#
# The Nimrod Compiler
# (c) Copyright 2008 Andreas Rumpf
# (c) Copyright 2011 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@ -13,29 +13,29 @@
import
intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
condsyms, ropes, idents, crc, rodread, passes, importer
condsyms, ropes, idents, crc, rodread, passes, importer, idgen, rodutils
proc rodwritePass*(): TPass
# implementation
type
TRodWriter = object of TPassContext
module*: PSym
crc*: TCrc32
options*: TOptions
defines*: PRope
inclDeps*: PRope
modDeps*: PRope
interf*: PRope
compilerProcs*: PRope
index*, imports*: TIndex
converters*: PRope
init*: PRope
data*: PRope
filename*: string
sstack*: TSymSeq # a stack of symbols to process
tstack*: TTypeSeq # a stack of types to process
files*: TStringSeq
module: PSym
crc: TCrc32
options: TOptions
defines: string
inclDeps: string
modDeps: string
interf: string
compilerProcs: string
index, imports: TIndex
converters: string
init: string
data: string
filename: string
sstack: TSymSeq # a stack of symbols to process
tstack: TTypeSeq # a stack of types to process
files: TStringSeq
PRodWriter = ref TRodWriter
@ -45,19 +45,17 @@ proc addInclDep(w: PRodWriter, dep: string)
proc addInterfaceSym(w: PRodWriter, s: PSym)
proc addStmt(w: PRodWriter, n: PNode)
proc writeRod(w: PRodWriter)
proc encodeStr(w: PRodWriter, s: string): PRope =
result = encode(s)
proc processStacks(w: PRodWriter)
proc getDefines(): PRope =
proc getDefines(): string =
var it: TTabIter
var s = InitTabIter(it, gSymbols)
result = nil
result = ""
while s != nil:
if s.position == 1:
if result != nil: app(result, " ")
app(result, s.name.s)
if result.len != 0: add(result, " ")
add(result, s.name.s)
s = nextIter(it, gSymbols)
proc fileIdx(w: PRodWriter, filename: string): int =
@ -74,205 +72,279 @@ proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter =
result.tstack = @[]
InitIITable(result.index.tab)
InitIITable(result.imports.tab)
result.index.r = ""
result.imports.r = ""
result.filename = modfilename
result.crc = crc
result.module = module
result.defines = getDefines()
result.options = options.gOptions
result.files = @[]
result.inclDeps = ""
result.modDeps = ""
result.interf = newStringOfCap(2_000)
result.compilerProcs = ""
result.converters = ""
result.init = ""
result.data = newStringOfCap(12_000)
proc addModDep(w: PRodWriter, dep: string) =
if w.modDeps != nil: app(w.modDeps, " ")
app(w.modDeps, encodeInt(fileIdx(w, dep)))
if w.modDeps.len != 0: add(w.modDeps, ' ')
encodeVInt(fileIdx(w, dep), w.modDeps)
const
rodNL = "\x0A"
proc addInclDep(w: PRodWriter, dep: string) =
app(w.inclDeps, encodeInt(fileIdx(w, dep)))
app(w.inclDeps, " ")
app(w.inclDeps, encodeInt(crcFromFile(dep)))
app(w.inclDeps, rodNL)
encodeVInt(fileIdx(w, dep), w.inclDeps)
add(w.inclDeps, " ")
encodeVInt(crcFromFile(dep), w.inclDeps)
add(w.inclDeps, rodNL)
proc pushType(w: PRodWriter, t: PType) =
proc pushType(w: PRodWriter, t: PType) =
# check so that the stack does not grow too large:
if IiTableGet(w.index.tab, t.id) == invalidKey:
var L = len(w.tstack)
setlen(w.tstack, L + 1)
w.tstack[L] = t
if IiTableGet(w.index.tab, t.id) == invalidKey:
w.tstack.add(t)
proc pushSym(w: PRodWriter, s: PSym) =
proc pushSym(w: PRodWriter, s: PSym) =
# check so that the stack does not grow too large:
if IiTableGet(w.index.tab, s.id) == invalidKey:
var L = len(w.sstack)
setlen(w.sstack, L + 1)
w.sstack[L] = s
if IiTableGet(w.index.tab, s.id) == invalidKey:
w.sstack.add(s)
proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode): PRope =
proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
result: var string) =
if n == nil:
# nil nodes have to be stored too:
return toRope("()")
result = toRope("(")
app(result, encodeInt(ord(n.kind)))
result.add("()")
return
result.add('(')
encodeVInt(ord(n.kind), result)
# we do not write comments for now
# Line information takes easily 20% or more of the filesize! Therefore we
# omit line information if it is the same as the father's line information:
if finfo.fileIndex != n.info.fileIndex:
appf(result, "?$1,$2,$3", [encodeInt(n.info.col), encodeInt(n.info.line),
encodeInt(fileIdx(w, toFilename(n.info)))])
result.add('?')
encodeVInt(n.info.col, result)
result.add(',')
encodeVInt(n.info.line, result)
result.add(',')
encodeVInt(fileIdx(w, toFilename(n.info)), result)
elif finfo.line != n.info.line:
appf(result, "?$1,$2", [encodeInt(n.info.col), encodeInt(n.info.line)])
result.add('?')
encodeVInt(n.info.col, result)
result.add(',')
encodeVInt(n.info.line, result)
elif finfo.col != n.info.col:
appf(result, "?$1", [encodeInt(n.info.col)])
result.add('?')
encodeVInt(n.info.col, result)
# No need to output the file index, as this is the serialization of one
# file.
var f = n.flags * PersistentNodeFlags
if f != {}: appf(result, "$$$1", [encodeInt(cast[int32](f))])
if n.typ != nil:
appf(result, "^$1", [encodeInt(n.typ.id)])
if f != {}:
result.add('$')
encodeVInt(cast[int32](f), result)
if n.typ != nil:
result.add('^')
encodeVInt(n.typ.id, result)
pushType(w, n.typ)
case n.kind
of nkCharLit..nkInt64Lit:
if n.intVal != 0: appf(result, "!$1", [encodeInt(n.intVal)])
if n.intVal != 0:
result.add('!')
encodeVBiggestInt(n.intVal, result)
of nkFloatLit..nkFloat64Lit:
if n.floatVal != 0.0: appf(result, "!$1", [encodeStr(w, $n.floatVal)])
of nkStrLit..nkTripleStrLit:
if n.strVal != "": appf(result, "!$1", [encodeStr(w, n.strVal)])
of nkIdent:
appf(result, "!$1", [encodeStr(w, n.ident.s)])
of nkSym:
appf(result, "!$1", [encodeInt(n.sym.id)])
if n.floatVal != 0.0:
result.add('!')
encodeStr($n.floatVal, result)
of nkStrLit..nkTripleStrLit:
if n.strVal != "":
result.add('!')
encodeStr(n.strVal, result)
of nkIdent:
result.add('!')
encodeStr(n.ident.s, result)
of nkSym:
result.add('!')
encodeVInt(n.sym.id, result)
pushSym(w, n.sym)
else:
else:
for i in countup(0, sonsLen(n) - 1):
app(result, encodeNode(w, n.info, n.sons[i]))
app(result, ")")
encodeNode(w, n.info, n.sons[i], result)
add(result, ')')
proc encodeLoc(w: PRodWriter, loc: TLoc): PRope =
result = nil
if loc.k != low(loc.k): app(result, encodeInt(ord(loc.k)))
if loc.s != low(loc.s): appf(result, "*$1", [encodeInt(ord(loc.s))])
if loc.flags != {}: appf(result, "$$$1", [encodeInt(cast[int32](loc.flags))])
if loc.t != nil:
appf(result, "^$1", [encodeInt(loc.t.id)])
proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
var oldLen = result.len
result.add('<')
if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
if loc.s != low(loc.s):
add(result, '*')
encodeVInt(ord(loc.s), result)
if loc.flags != {}:
add(result, '$')
encodeVInt(cast[int32](loc.flags), result)
if loc.t != nil:
add(result, '^')
encodeVInt(cast[int32](loc.t.id), result)
pushType(w, loc.t)
if loc.r != nil: appf(result, "!$1", [encodeStr(w, ropeToStr(loc.r))])
if loc.a != 0: appf(result, "?$1", [encodeInt(loc.a)])
if result != nil: result = ropef("<$1>", [result])
if loc.r != nil:
add(result, '!')
encodeStr(ropeToStr(loc.r), result)
if loc.a != 0:
add(result, '?')
encodeVInt(loc.a, result)
if oldlen + 1 == result.len:
# no data was necessary, so remove the '<' again:
setLen(result, oldLen)
else:
add(result, '>')
proc encodeType(w: PRodWriter, t: PType): PRope =
proc encodeType(w: PRodWriter, t: PType, result: var string) =
if t == nil:
# nil nodes have to be stored too:
return toRope("[]")
result = nil
result.add("[]")
return
# we need no surrounding [] here because the type is in a line of its own
if t.kind == tyForward: InternalError("encodeType: tyForward")
app(result, encodeInt(ord(t.kind)))
appf(result, "+$1", [encodeInt(t.id)])
if t.n != nil: app(result, encodeNode(w, UnknownLineInfo(), t.n))
if t.flags != {}: appf(result, "$$$1", [encodeInt(cast[int32](t.flags))])
encodeVInt(ord(t.kind), result)
add(result, '+')
encodeVInt(t.id, result)
if t.n != nil:
encodeNode(w, UnknownLineInfo(), t.n, result)
if t.flags != {}:
add(result, '$')
encodeVInt(cast[int32](t.flags), result)
if t.callConv != low(t.callConv):
appf(result, "?$1", [encodeInt(ord(t.callConv))])
add(result, '?')
encodeVInt(ord(t.callConv), result)
if t.owner != nil:
appf(result, "*$1", [encodeInt(t.owner.id)])
add(result, '*')
encodeVInt(t.owner.id, result)
pushSym(w, t.owner)
if t.sym != nil:
appf(result, "&$1", [encodeInt(t.sym.id)])
add(result, '&')
encodeVInt(t.sym.id, result)
pushSym(w, t.sym)
if t.size != - 1: appf(result, "/$1", [encodeInt(t.size)])
if t.align != 2: appf(result, "=$1", [encodeInt(t.align)])
if t.containerID != 0: appf(result, "@$1", [encodeInt(t.containerID)])
app(result, encodeLoc(w, t.loc))
if t.size != - 1:
add(result, '/')
encodeVBiggestInt(t.size, result)
if t.align != 2:
add(result, '=')
encodeVInt(t.align, result)
if t.containerID != 0:
add(result, '@')
encodeVInt(t.containerID, result)
encodeLoc(w, t.loc, result)
for i in countup(0, sonsLen(t) - 1):
if t.sons[i] == nil:
app(result, "^()")
add(result, "^()")
else:
appf(result, "^$1", [encodeInt(t.sons[i].id)])
add(result, '^')
encodeVInt(t.sons[i].id, result)
pushType(w, t.sons[i])
proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo): PRope =
result = nil
appf(result, "|$1", [encodeInt(ord(lib.kind))])
appf(result, "|$1", [encodeStr(w, ropeToStr(lib.name))])
appf(result, "|$1", [encodeNode(w, info, lib.path)])
proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
add(result, '|')
encodeVInt(ord(lib.kind), result)
add(result, '|')
encodeStr(ropeToStr(lib.name), result)
add(result, '|')
encodeNode(w, info, lib.path, result)
proc encodeSym(w: PRodWriter, s: PSym): PRope =
var
codeAst: PNode
col, line: PRope
codeAst = nil
if s == nil:
proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
if s == nil:
# nil nodes have to be stored too:
return toRope("{}")
result = nil
app(result, encodeInt(ord(s.kind)))
appf(result, "+$1", [encodeInt(s.id)])
appf(result, "&$1", [encodeStr(w, s.name.s)])
if s.typ != nil:
appf(result, "^$1", [encodeInt(s.typ.id)])
result.add("{}")
return
# we need no surrounding {} here because the symbol is in a line of its own
encodeVInt(ord(s.kind), result)
result.add('+')
encodeVInt(s.id, result)
result.add('&')
encodeStr(s.name.s, result)
if s.typ != nil:
result.add('^')
encodeVInt(s.typ.id, result)
pushType(w, s.typ)
if s.info.col == int16(- 1): col = nil
else: col = encodeInt(s.info.col)
if s.info.line == int16(- 1): line = nil
else: line = encodeInt(s.info.line)
appf(result, "?$1,$2,$3",
[col, line, encodeInt(fileIdx(w, toFilename(s.info)))])
if s.owner != nil:
appf(result, "*$1", [encodeInt(s.owner.id)])
result.add('?')
if s.info.col != -1'i16: encodeVInt(s.info.col, result)
result.add(',')
if s.info.line != -1'i16: encodeVInt(s.info.line, result)
result.add(',')
encodeVInt(fileIdx(w, toFilename(s.info)), result)
if s.owner != nil:
result.add('*')
encodeVInt(s.owner.id, result)
pushSym(w, s.owner)
if s.flags != {}: appf(result, "$$$1", [encodeInt(cast[int32](s.flags))])
if s.magic != mNone: appf(result, "@$1", [encodeInt(ord(s.magic))])
if (s.ast != nil):
if not astNeeded(s):
if s.flags != {}:
result.add('$')
encodeVInt(cast[int32](s.flags), result)
if s.magic != mNone:
result.add('@')
encodeVInt(ord(s.magic), result)
if s.ast != nil:
var codeAst: PNode = nil
if not astNeeded(s):
codeAst = s.ast.sons[codePos]
# ugly hack to not store the AST:
s.ast.sons[codePos] = nil
app(result, encodeNode(w, s.info, s.ast))
if codeAst != nil:
encodeNode(w, s.info, s.ast, result)
if codeAst != nil:
# resore the AST:
s.ast.sons[codePos] = codeAst
if s.options != w.options:
appf(result, "!$1", [encodeInt(cast[int32](s.options))])
if s.position != 0: appf(result, "%$1", [encodeInt(s.position)])
if s.offset != - 1: appf(result, "`$1", [encodeInt(s.offset)])
app(result, encodeLoc(w, s.loc))
if s.annex != nil: app(result, encodeLib(w, s.annex, s.info))
result.add('!')
encodeVInt(cast[int32](s.options), result)
if s.position != 0:
result.add('%')
encodeVInt(s.position, result)
if s.offset != - 1:
result.add('`')
encodeVInt(s.offset, result)
encodeLoc(w, s.loc, result)
if s.annex != nil: encodeLib(w, s.annex, s.info, result)
proc addToIndex(w: var TIndex, key, val: int) =
if key - w.lastIdxKey == 1:
proc addToIndex(w: var TIndex, key, val: int) =
if key - w.lastIdxKey == 1:
# we do not store a key-diff of 1 to safe space
app(w.r, encodeInt(val - w.lastIdxVal))
app(w.r, rodNL)
else:
appf(w.r, "$1 $2" & rodNL,
[encodeInt(key - w.lastIdxKey), encodeInt(val - w.lastIdxVal)])
encodeVInt(val - w.lastIdxVal, w.r)
else:
encodeVInt(key - w.lastIdxKey, w.r)
add(w.r, ' ')
encodeVInt(val - w.lastIdxVal, w.r)
add(w.r, rodNL)
w.lastIdxKey = key
w.lastIdxVal = val
IiTablePut(w.tab, key, val)
var debugWritten: TIntSet
proc symStack(w: PRodWriter) =
var
i, L: int
s, m: PSym
i = 0
proc symStack(w: PRodWriter) =
var i = 0
while i < len(w.sstack):
s = w.sstack[i]
var s = w.sstack[i]
if IiTableGet(w.index.tab, s.id) == invalidKey:
m = getModule(s)
var m = getModule(s)
if m == nil: InternalError("symStack: module nil: " & s.name.s)
if (m.id == w.module.id) or (sfFromGeneric in s.flags):
# put definition in here
L = ropeLen(w.data)
addToIndex(w.index, s.id, L) #intSetIncl(debugWritten, s.id);
app(w.data, encodeSym(w, s))
app(w.data, rodNL)
var L = w.data.len
addToIndex(w.index, s.id, L)
#intSetIncl(debugWritten, s.id)
encodeSym(w, s, w.data)
add(w.data, rodNL)
if sfExported in s.flags:
appf(w.interf, "$1 $2" & rodNL, [encode(s.name.s), encodeInt(s.id)])
if sfCompilerProc in s.flags:
appf(w.compilerProcs, "$1 $2" & rodNL,
[encode(s.name.s), encodeInt(s.id)])
encodeStr(s.name.s, w.interf)
add(w.interf, ' ')
encodeVInt(s.id, w.interf)
add(w.interf, rodNL)
if sfCompilerProc in s.flags:
encodeStr(s.name.s, w.compilerProcs)
add(w.compilerProcs, ' ')
encodeVInt(s.id, w.compilerProcs)
add(w.compilerProcs, rodNL)
if s.kind == skConverter:
if w.converters != nil: app(w.converters, " ")
app(w.converters, encodeInt(s.id))
if w.converters.len != 0: add(w.converters, ' ')
encodeVInt(s.id, w.converters)
elif IiTableGet(w.imports.tab, s.id) == invalidKey:
addToIndex(w.imports, s.id, m.id) #if not Contains(debugWritten, s.id):
# MessageOut(w.filename);
@ -287,10 +359,10 @@ proc typeStack(w: PRodWriter) =
var i = 0
while i < len(w.tstack):
if IiTableGet(w.index.tab, w.tstack[i].id) == invalidKey:
var L = ropeLen(w.data)
var L = w.data.len
addToIndex(w.index, w.tstack[i].id, L)
app(w.data, encodeType(w, w.tstack[i]))
app(w.data, rodNL)
encodeType(w, w.tstack[i], w.data)
add(w.data, rodNL)
inc(i)
setlen(w.tstack, 0)
@ -309,66 +381,88 @@ proc addInterfaceSym(w: PRodWriter, s: PSym) =
rawAddInterfaceSym(w, s)
proc addStmt(w: PRodWriter, n: PNode) =
app(w.init, encodeInt(ropeLen(w.data)))
app(w.init, rodNL)
app(w.data, encodeNode(w, UnknownLineInfo(), n))
app(w.data, rodNL)
encodeVInt(w.data.len, w.init)
add(w.init, rodNL)
encodeNode(w, UnknownLineInfo(), n, w.data)
add(w.data, rodNL)
processStacks(w)
proc writeRod(w: PRodWriter) =
processStacks(w) # write header:
var content = toRope("NIM:")
app(content, toRope(FileVersion))
app(content, rodNL)
app(content, toRope("ID:"))
app(content, encodeInt(w.module.id))
app(content, rodNL)
app(content, toRope("CRC:"))
app(content, encodeInt(w.crc))
app(content, rodNL)
app(content, toRope("OPTIONS:"))
app(content, encodeInt(cast[int32](w.options)))
app(content, rodNL)
app(content, toRope("DEFINES:"))
app(content, w.defines)
app(content, rodNL)
app(content, toRope("FILES(" & rodNL))
processStacks(w)
var f: TFile
if not open(f, completeGeneratedFilePath(changeFileExt(w.filename, "rod")),
fmWrite):
return
# write header:
f.write("NIM:")
f.write(FileVersion)
f.write(rodNL)
var id = "ID:"
encodeVInt(w.module.id, id)
f.write(id)
f.write(rodNL)
var crc = "CRC:"
encodeVInt(w.crc, crc)
f.write(crc)
f.write(rodNL)
var options = "OPTIONS:"
encodeVInt(cast[int32](w.options), options)
f.write(options)
f.write(rodNL)
f.write("DEFINES:")
f.write(w.defines)
f.write(rodNL)
var files = "FILES(" & rodNL
for i in countup(0, high(w.files)):
app(content, encode(w.files[i]))
app(content, rodNL)
app(content, toRope(')' & rodNL))
app(content, toRope("INCLUDES(" & rodNL))
app(content, w.inclDeps)
app(content, toRope(')' & rodNL))
app(content, toRope("DEPS:"))
app(content, w.modDeps)
app(content, rodNL)
app(content, toRope("INTERF(" & rodNL))
app(content, w.interf)
app(content, toRope(')' & rodNL))
app(content, toRope("COMPILERPROCS(" & rodNL))
app(content, w.compilerProcs)
app(content, toRope(')' & rodNL))
app(content, toRope("INDEX(" & rodNL))
app(content, w.index.r)
app(content, toRope(')' & rodNL))
app(content, toRope("IMPORTS(" & rodNL))
app(content, w.imports.r)
app(content, toRope(')' & rodNL))
app(content, toRope("CONVERTERS:"))
app(content, w.converters)
app(content, toRope(rodNL))
app(content, toRope("INIT(" & rodNL))
app(content, w.init)
app(content, toRope(')' & rodNL))
app(content, toRope("DATA(" & rodNL))
app(content, w.data)
app(content, toRope(')' & rodNL))
#MessageOut('interf ' + ToString(ropeLen(w.interf)));
#MessageOut('index ' + ToString(ropeLen(w.indexRope)));
#MessageOut('init ' + ToString(ropeLen(w.init)));
#MessageOut('data ' + ToString(ropeLen(w.data)));
writeRope(content, completeGeneratedFilePath(changeFileExt(w.filename, "rod")))
encodeStr(w.files[i], files)
files.add(rodNL)
f.write(files)
f.write(')' & rodNL)
f.write("INCLUDES(" & rodNL)
f.write(w.inclDeps)
f.write(')' & rodNL)
f.write("DEPS:")
f.write(w.modDeps)
f.write(rodNL)
f.write("INTERF(" & rodNL)
f.write(w.interf)
f.write(')' & rodNL)
f.write("COMPILERPROCS(" & rodNL)
f.write(w.compilerProcs)
f.write(')' & rodNL)
f.write("INDEX(" & rodNL)
f.write(w.index.r)
f.write(')' & rodNL)
f.write("IMPORTS(" & rodNL)
f.write(w.imports.r)
f.write(')' & rodNL)
f.write("CONVERTERS:")
f.write(w.converters)
f.write(rodNL)
f.write("INIT(" & rodNL)
f.write(w.init)
f.write(')' & rodNL)
f.write("DATA(" & rodNL)
f.write(w.data)
f.write(')' & rodNL)
f.close()
#MessageOut('interf ' + ToString(ropeLen(w.interf)))
#MessageOut('index ' + ToString(ropeLen(w.indexRope)))
#MessageOut('init ' + ToString(ropeLen(w.init)))
#MessageOut('data ' + ToString(ropeLen(w.data)))
proc process(c: PPassContext, n: PNode): PNode =
result = n
@ -438,6 +532,7 @@ proc myClose(c: PPassContext, n: PNode): PNode =
var w = PRodWriter(c)
writeRod(w)
result = n
idgen.saveMaxIds(options.projectPath / options.projectName)
proc rodwritePass(): TPass =
initPass(result)
@ -446,4 +541,4 @@ proc rodwritePass(): TPass =
result.close = myClose
result.process = process
debugWritten= initIntSet()
debugWritten = initIntSet()