much more efficient rod file generation

This commit is contained in:
Araq 2011-10-18 17:21:51 +02:00
commit 4de84024e5
18 changed files with 611 additions and 557 deletions

View file

@ -73,7 +73,7 @@
import
os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms,
ropes, idents, crc
ropes, idents, crc, idgen, rodutils
type
TReasonForRecompile* = enum
@ -100,12 +100,12 @@ type
TIndex*{.final.} = object # an index with compression
lastIdxKey*, lastIdxVal*: int
tab*: TIITable
r*: PRope # writers use this
r*: string # writers use this
offset*: int # readers use this
TRodReader* = object of TObject
pos*: int # position; used for parsing
s*: string # the whole file in memory
s*: string # the whole file in memory; XXX mmap this!
options*: TOptions
reason*: TReasonForRecompile
modDeps*: TStringSeq
@ -116,7 +116,7 @@ type
filename*: string
index*, imports*: TIndex
readerIndex*: int
line*: int # only used for debugging, but is always in the code
line*: int # only used for debugging, but is always in the code
moduleID*: int
syms*: TIdTable # already processed symbols
@ -132,8 +132,7 @@ proc handleSymbolFile*(module: PSym, filename: string): PRodReader
proc GetCRC*(filename: string): TCrc32
proc loadInitSection*(r: PRodReader): PNode
proc loadStub*(s: PSym)
proc encodeInt*(x: BiggestInt): PRope
proc encode*(s: string): PRope
# implementation
var gTypeTable: TIdTable
@ -141,52 +140,19 @@ var gTypeTable: TIdTable
proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym
# `info` is only used for debugging purposes
proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType
proc decode(r: PRodReader): string
proc decodeInt(r: PRodReader): int
proc decodeBInt(r: PRodReader): biggestInt
proc encode(s: string): PRope =
var res = ""
for i in countup(0, len(s) - 1):
case s[i]
of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(res, s[i])
else: add(res, '\\' & toHex(ord(s[i]), 2))
result = toRope(res)
proc encodeIntAux(str: var string, x: BiggestInt) =
const chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
var d: char
var v = x
var rem: biggestInt = v mod 190
if (rem < 0):
add(str, '-')
v = - (v div 190)
rem = - rem
else:
v = v div 190
var idx = int(rem)
if idx < 62: d = chars[idx + 0]
else: d = chr(idx - 62 + 128)
if (v != 0): encodeIntAux(str, v)
add(str, d)
proc encodeInt(x: BiggestInt): PRope =
var res = ""
encodeIntAux(res, x)
result = toRope(res)
proc decodeLineInfo(r: PRodReader, info: var TLineInfo) =
if r.s[r.pos] == '?':
inc(r.pos)
if r.s[r.pos] == ',': info.col = int16(- 1)
else: info.col = int16(decodeInt(r))
else: info.col = int16(decodeVInt(r.s, r.pos))
if r.s[r.pos] == ',':
inc(r.pos)
if r.s[r.pos] == ',': info.line = int16(- 1)
else: info.line = int16(decodeInt(r))
else: info.line = int16(decodeVInt(r.s, r.pos))
if r.s[r.pos] == ',':
inc(r.pos)
info = newLineInfo(r.files[decodeInt(r)], info.line, info.col)
info = newLineInfo(r.files[decodeVInt(r.s, r.pos)], info.line, info.col)
proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode =
result = nil
@ -195,42 +161,42 @@ proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode =
if r.s[r.pos] == ')':
inc(r.pos)
return # nil node
result = newNodeI(TNodeKind(decodeInt(r)), fInfo)
result = newNodeI(TNodeKind(decodeVInt(r.s, r.pos)), fInfo)
decodeLineInfo(r, result.info)
if r.s[r.pos] == '$':
inc(r.pos)
result.flags = cast[TNodeFlags](int32(decodeInt(r)))
result.flags = cast[TNodeFlags](int32(decodeVInt(r.s, r.pos)))
if r.s[r.pos] == '^':
inc(r.pos)
var id = decodeInt(r)
var id = decodeVInt(r.s, r.pos)
result.typ = rrGetType(r, id, result.info)
case result.kind
of nkCharLit..nkInt64Lit:
if r.s[r.pos] == '!':
inc(r.pos)
result.intVal = decodeBInt(r)
result.intVal = decodeVBiggestInt(r.s, r.pos)
of nkFloatLit..nkFloat64Lit:
if r.s[r.pos] == '!':
inc(r.pos)
var fl = decode(r)
var fl = decodeStr(r.s, r.pos)
result.floatVal = parseFloat(fl)
of nkStrLit..nkTripleStrLit:
if r.s[r.pos] == '!':
inc(r.pos)
result.strVal = decode(r)
result.strVal = decodeStr(r.s, r.pos)
else:
result.strVal = "" # BUGFIX
of nkIdent:
if r.s[r.pos] == '!':
inc(r.pos)
var fl = decode(r)
var fl = decodeStr(r.s, r.pos)
result.ident = getIdent(fl)
else:
internalError(result.info, "decodeNode: nkIdent")
of nkSym:
if r.s[r.pos] == '!':
inc(r.pos)
var id = decodeInt(r)
var id = decodeVInt(r.s, r.pos)
result.sym = rrGetSym(r, id, result.info)
else:
internalError(result.info, "decodeNode: nkSym")
@ -245,32 +211,32 @@ proc decodeLoc(r: PRodReader, loc: var TLoc, info: TLineInfo) =
if r.s[r.pos] == '<':
inc(r.pos)
if r.s[r.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}:
loc.k = TLocKind(decodeInt(r))
loc.k = TLocKind(decodeVInt(r.s, r.pos))
else:
loc.k = low(loc.k)
if r.s[r.pos] == '*':
inc(r.pos)
loc.s = TStorageLoc(decodeInt(r))
loc.s = TStorageLoc(decodeVInt(r.s, r.pos))
else:
loc.s = low(loc.s)
if r.s[r.pos] == '$':
inc(r.pos)
loc.flags = cast[TLocFlags](int32(decodeInt(r)))
loc.flags = cast[TLocFlags](int32(decodeVInt(r.s, r.pos)))
else:
loc.flags = {}
if r.s[r.pos] == '^':
inc(r.pos)
loc.t = rrGetType(r, decodeInt(r), info)
loc.t = rrGetType(r, decodeVInt(r.s, r.pos), info)
else:
loc.t = nil
if r.s[r.pos] == '!':
inc(r.pos)
loc.r = toRope(decode(r))
loc.r = toRope(decodeStr(r.s, r.pos))
else:
loc.r = nil
if r.s[r.pos] == '?':
inc(r.pos)
loc.a = decodeInt(r)
loc.a = decodeVInt(r.s, r.pos)
else:
loc.a = 0
if r.s[r.pos] == '>': inc(r.pos)
@ -284,10 +250,10 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
inc(r.pos)
return # nil type
new(result)
result.kind = TTypeKind(decodeInt(r))
result.kind = TTypeKind(decodeVInt(r.s, r.pos))
if r.s[r.pos] == '+':
inc(r.pos)
result.id = decodeInt(r)
result.id = decodeVInt(r.s, r.pos)
setId(result.id)
if debugIds: registerID(result)
else:
@ -297,29 +263,29 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
if r.s[r.pos] == '(': result.n = decodeNode(r, UnknownLineInfo())
if r.s[r.pos] == '$':
inc(r.pos)
result.flags = cast[TTypeFlags](int32(decodeInt(r)))
result.flags = cast[TTypeFlags](int32(decodeVInt(r.s, r.pos)))
if r.s[r.pos] == '?':
inc(r.pos)
result.callConv = TCallingConvention(decodeInt(r))
result.callConv = TCallingConvention(decodeVInt(r.s, r.pos))
if r.s[r.pos] == '*':
inc(r.pos)
result.owner = rrGetSym(r, decodeInt(r), info)
result.owner = rrGetSym(r, decodeVInt(r.s, r.pos), info)
if r.s[r.pos] == '&':
inc(r.pos)
result.sym = rrGetSym(r, decodeInt(r), info)
result.sym = rrGetSym(r, decodeVInt(r.s, r.pos), info)
if r.s[r.pos] == '/':
inc(r.pos)
result.size = decodeInt(r)
result.size = decodeVInt(r.s, r.pos)
else:
result.size = - 1
if r.s[r.pos] == '=':
inc(r.pos)
result.align = decodeInt(r)
result.align = decodeVInt(r.s, r.pos)
else:
result.align = 2
if r.s[r.pos] == '@':
inc(r.pos)
result.containerID = decodeInt(r)
result.containerID = decodeVInt(r.s, r.pos)
decodeLoc(r, result.loc, info)
while r.s[r.pos] == '^':
inc(r.pos)
@ -329,7 +295,7 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
else: InternalError(info, "decodeType ^(" & r.s[r.pos])
addSon(result, nil)
else:
var d = decodeInt(r)
var d = decodeVInt(r.s, r.pos)
addSon(result, rrGetType(r, d, info))
proc decodeLib(r: PRodReader, info: TLineInfo): PLib =
@ -337,10 +303,10 @@ proc decodeLib(r: PRodReader, info: TLineInfo): PLib =
if r.s[r.pos] == '|':
new(result)
inc(r.pos)
result.kind = TLibKind(decodeInt(r))
result.kind = TLibKind(decodeVInt(r.s, r.pos))
if r.s[r.pos] != '|': InternalError("decodeLib: 1")
inc(r.pos)
result.name = toRope(decode(r))
result.name = toRope(decodeStr(r.s, r.pos))
if r.s[r.pos] != '|': InternalError("decodeLib: 2")
inc(r.pos)
result.path = decodeNode(r, info)
@ -355,16 +321,16 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
if r.s[r.pos] == '}':
inc(r.pos)
return # nil sym
var k = TSymKind(decodeInt(r))
var k = TSymKind(decodeVInt(r.s, r.pos))
if r.s[r.pos] == '+':
inc(r.pos)
id = decodeInt(r)
id = decodeVInt(r.s, r.pos)
setId(id)
else:
InternalError(info, "decodeSym: no id")
if r.s[r.pos] == '&':
inc(r.pos)
ident = getIdent(decode(r))
ident = getIdent(decodeStr(r.s, r.pos))
else:
InternalError(info, "decodeSym: no ident")
result = PSym(IdTableGet(r.syms, id))
@ -379,100 +345,37 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
result.name = ident # read the rest of the symbol description:
if r.s[r.pos] == '^':
inc(r.pos)
result.typ = rrGetType(r, decodeInt(r), info)
result.typ = rrGetType(r, decodeVInt(r.s, r.pos), info)
decodeLineInfo(r, result.info)
if r.s[r.pos] == '*':
inc(r.pos)
result.owner = rrGetSym(r, decodeInt(r), result.info)
result.owner = rrGetSym(r, decodeVInt(r.s, r.pos), result.info)
if r.s[r.pos] == '$':
inc(r.pos)
result.flags = cast[TSymFlags](int32(decodeInt(r)))
result.flags = cast[TSymFlags](int32(decodeVInt(r.s, r.pos)))
if r.s[r.pos] == '@':
inc(r.pos)
result.magic = TMagic(decodeInt(r))
result.magic = TMagic(decodeVInt(r.s, r.pos))
if r.s[r.pos] == '(': result.ast = decodeNode(r, result.info)
if r.s[r.pos] == '!':
inc(r.pos)
result.options = cast[TOptions](int32(decodeInt(r)))
result.options = cast[TOptions](int32(decodeVInt(r.s, r.pos)))
else:
result.options = r.options
if r.s[r.pos] == '%':
inc(r.pos)
result.position = decodeInt(r)
result.position = decodeVInt(r.s, r.pos)
else:
result.position = 0
# BUGFIX: this may have been misused as reader index!
if r.s[r.pos] == '`':
inc(r.pos)
result.offset = decodeInt(r)
result.offset = decodeVInt(r.s, r.pos)
else:
result.offset = - 1
decodeLoc(r, result.loc, result.info)
result.annex = decodeLib(r, info)
proc decodeInt(r: PRodReader): int =
# base 190 numbers
var i = r.pos
var sign = - 1
assert(r.s[i] in {'a'..'z', 'A'..'Z', '0'..'9', '-', '\x80'..'\xFF'})
if r.s[i] == '-':
inc(i)
sign = 1
result = 0
while true:
case r.s[i]
of '0'..'9': result = result * 190 - (ord(r.s[i]) - ord('0'))
of 'a'..'z': result = result * 190 - (ord(r.s[i]) - ord('a') + 10)
of 'A'..'Z': result = result * 190 - (ord(r.s[i]) - ord('A') + 36)
of '\x80'..'\xFF': result = result * 190 - (ord(r.s[i]) - 128 + 62)
else: break
inc(i)
result = result * sign
r.pos = i
proc decodeBInt(r: PRodReader): biggestInt =
var i = r.pos
var sign: biggestInt = - 1
assert(r.s[i] in {'a'..'z', 'A'..'Z', '0'..'9', '-', '\x80'..'\xFF'})
if r.s[i] == '-':
inc(i)
sign = 1
result = 0
while true:
case r.s[i]
of '0'..'9': result = result * 190 - (ord(r.s[i]) - ord('0'))
of 'a'..'z': result = result * 190 - (ord(r.s[i]) - ord('a') + 10)
of 'A'..'Z': result = result * 190 - (ord(r.s[i]) - ord('A') + 36)
of '\x80'..'\xFF': result = result * 190 - (ord(r.s[i]) - 128 + 62)
else: break
inc(i)
result = result * sign
r.pos = i
proc hexChar(c: char, xi: var int) =
case c
of '0'..'9': xi = (xi shl 4) or (ord(c) - ord('0'))
of 'a'..'f': xi = (xi shl 4) or (ord(c) - ord('a') + 10)
of 'A'..'F': xi = (xi shl 4) or (ord(c) - ord('A') + 10)
else: nil
proc decode(r: PRodReader): string =
var i = r.pos
result = ""
while true:
case r.s[i]
of '\\':
inc(i, 3)
var xi = 0
hexChar(r.s[i-2], xi)
hexChar(r.s[i-1], xi)
add(result, chr(xi))
of 'a'..'z', 'A'..'Z', '0'..'9', '_':
add(result, r.s[i])
inc(i)
else: break
r.pos = i
proc skipSection(r: PRodReader) =
if r.s[r.pos] == ':':
while r.s[r.pos] > '\x0A': inc(r.pos)
@ -514,9 +417,9 @@ proc processInterf(r: PRodReader, module: PSym) =
if r.interfIdx == 0: InternalError("processInterf")
r.pos = r.interfIdx
while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'):
var w = decode(r)
var w = decodeStr(r.s, r.pos)
inc(r.pos)
var key = decodeInt(r)
var key = decodeVInt(r.s, r.pos)
inc(r.pos) # #10
var s = newStub(r, w, key)
s.owner = module
@ -527,9 +430,9 @@ proc processCompilerProcs(r: PRodReader, module: PSym) =
if r.compilerProcsIdx == 0: InternalError("processCompilerProcs")
r.pos = r.compilerProcsIdx
while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'):
var w = decode(r)
var w = decodeStr(r.s, r.pos)
inc(r.pos)
var key = decodeInt(r)
var key = decodeVInt(r.s, r.pos)
inc(r.pos) # #10
var s = PSym(IdTableGet(r.syms, key))
if s == nil:
@ -543,11 +446,11 @@ proc processIndex(r: PRodReader, idx: var TIndex) =
inc(r.pos, 2) # skip "(\10"
inc(r.line)
while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'):
tmp = decodeInt(r)
tmp = decodeVInt(r.s, r.pos)
if r.s[r.pos] == ' ':
inc(r.pos)
key = idx.lastIdxKey + tmp
val = decodeInt(r) + idx.lastIdxVal
val = decodeVInt(r.s, r.pos) + idx.lastIdxVal
else:
key = idx.lastIdxKey + 1
val = tmp + idx.lastIdxVal
@ -571,20 +474,20 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
case section
of "CRC":
inc(r.pos) # skip ':'
if int(crc) != decodeInt(r): r.reason = rrCrcChange
if int(crc) != decodeVInt(r.s, r.pos): r.reason = rrCrcChange
of "ID":
inc(r.pos) # skip ':'
r.moduleID = decodeInt(r)
r.moduleID = decodeVInt(r.s, r.pos)
setID(r.moduleID)
of "OPTIONS":
inc(r.pos) # skip ':'
r.options = cast[TOptions](int32(decodeInt(r)))
r.options = cast[TOptions](int32(decodeVInt(r.s, r.pos)))
if options.gOptions != r.options: r.reason = rrOptions
of "DEFINES":
inc(r.pos) # skip ':'
d = 0
while r.s[r.pos] > '\x0A':
w = decode(r)
w = decodeStr(r.s, r.pos)
inc(d)
if not condsyms.isDefined(getIdent(w)):
r.reason = rrDefines #MessageOut('not defined, but should: ' + w);
@ -596,7 +499,7 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
L = 0
while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')':
setlen(r.files, L + 1)
r.files[L] = decode(r)
r.files[L] = decodeStr(r.s, r.pos)
inc(r.pos) # skip #10
inc(r.line)
inc(L)
@ -605,9 +508,9 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
inc(r.pos, 2) # skip "(\10"
inc(r.line)
while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')':
w = r.files[decodeInt(r)]
w = r.files[decodeVInt(r.s, r.pos)]
inc(r.pos) # skip ' '
inclCrc = decodeInt(r)
inclCrc = decodeVInt(r.s, r.pos)
if r.reason == rrNone:
if not ExistsFile(w) or (inclCrc != int(crcFromFile(w))):
r.reason = rrInclDeps
@ -620,7 +523,7 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
L = 0
while r.s[r.pos] > '\x0A':
setlen(r.modDeps, L + 1)
r.modDeps[L] = r.files[decodeInt(r)]
r.modDeps[L] = r.files[decodeVInt(r.s, r.pos)]
inc(L)
if r.s[r.pos] == ' ': inc(r.pos)
of "INTERF":
@ -723,8 +626,9 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
if d == invalidKey:
var moduleID = IiTableGet(r.imports.tab, id)
if moduleID < 0:
InternalError(info,
"missing from both indexes: +" & ropeToStr(encodeInt(id)))
var x = ""
encodeVInt(id, x)
InternalError(info, "missing from both indexes: +" & x)
# find the reader with the correct moduleID:
for i in countup(0, high(gMods)):
var rd = gMods[i].rd
@ -734,10 +638,11 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
if d != invalidKey:
result = decodeSymSafePos(rd, d, info)
break
else:
InternalError(info, "rrGetSym: no reader found: +" &
ropeToStr(encodeInt(id)))
else:
else:
var x = ""
encodeVInt(id, x)
InternalError(info, "rrGetSym: no reader found: +" & x)
else:
#if IiTableGet(rd.index.tab, id) <> invalidKey then
# XXX expensive check!
#InternalError(info,
@ -753,7 +658,7 @@ proc loadInitSection(r: PRodReader): PNode =
r.pos = r.initIdx
result = newNode(nkStmtList)
while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')':
var d = decodeInt(r)
var d = decodeVInt(r.s, r.pos)
inc(r.pos) # #10
var p = r.pos
r.pos = d + r.dataIdx
@ -767,7 +672,7 @@ proc loadConverters(r: PRodReader) =
InternalError("importConverters")
r.pos = r.convertersIdx
while (r.s[r.pos] > '\x0A'):
var d = decodeInt(r)
var d = decodeVInt(r.s, r.pos)
discard rrGetSym(r, d, UnknownLineInfo())
if r.s[r.pos] == ' ': inc(r.pos)
@ -818,9 +723,11 @@ proc checkDep(filename: string): TReasonForRecompile =
gMods[idx].reason = result # now we know better
proc handleSymbolFile(module: PSym, filename: string): PRodReader =
if not (optSymbolFiles in gGlobalOptions):
if optSymbolFiles notin gGlobalOptions:
module.id = getID()
return nil
idgen.loadMaxIds(options.projectPath / options.projectName)
discard checkDep(filename)
var idx = getModuleIdx(filename)
if gMods[idx].reason == rrEmpty: InternalError("handleSymbolFile")