jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

incremental compilation subsystem compiles again

Browse source
This commit is contained in:
Andreas Rumpf 2018-05-30 20:27:41 +02:00
commit a36c779f39
3 changed files with 149 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

26
compiler/incremental.nim
View file

@ -15,7 +15,8 @@ const nimIncremental* = defined(nimIncremental)
import options, lineinfos import options, lineinfos
when nimIncremental: when nimIncremental:
import ast, intsets, btrees, db_sqlite import ast, msgs, intsets, btrees, db_sqlite, std / sha1
from strutils import parseInt
type type
Writer* = object Writer* = object
@ -44,30 +45,29 @@ when nimIncremental:
proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: string): string = proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: string): string =
result = msgs.getHash(fileIdx) result = msgs.getHash(conf, fileIdx)
if result.len == 0: if result.len == 0:
result = $secureHashFile(fullpath) result = $secureHashFile(fullpath)
msgs.setHash(fileIdx, result) msgs.setHash(conf, fileIdx, result)
proc toDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; fileIdx: FileIndex): int =
proc toDbFileId*(fileIdx: int32): int = if fileIdx == FileIndex(-1): return -1
if fileIdx == -1: return -1 let fullpath = toFullPath(conf, fileIdx)
let fullpath = fileIdx.toFullPath let row = incr.db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
let row = db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
fullpath) fullpath)
let id = row[0] let id = row[0]
let fullhash = hashFileCached(fileIdx, fullpath) let fullhash = hashFileCached(conf, fileIdx, fullpath)
if id.len == 0: if id.len == 0:
result = int db.insertID(sql"insert into filenames(fullpath, fullhash) values (?, ?)", result = int incr.db.insertID(sql"insert into filenames(fullpath, fullhash) values (?, ?)",
fullpath, fullhash) fullpath, fullhash)
else: else:
if row[1] != fullhash: if row[1] != fullhash:
db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath) incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
result = parseInt(id) result = parseInt(id)
proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex = proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex =
if dbId == -1: return -1 if dbId == -1: return FileIndex(-1)
let fullpath = db.getValue(sql"select fullpath from filenames where id = ?", dbId) let fullpath = incr.db.getValue(sql"select fullpath from filenames where id = ?", dbId)
doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
result = fileInfoIdx(conf, fullpath) result = fileInfoIdx(conf, fullpath)

1
compiler/nim.cfg
View file

@ -6,6 +6,7 @@ path:"$projectPath/.."
define:booting define:booting
define:nimcore define:nimcore
define:nimIncremental
#import:"$projectpath/testability" #import:"$projectpath/testability"
@if windows: @if windows:

310
compiler/rodimpl.nim
View file

@ -10,7 +10,7 @@
## This module implements the new compilation cache. ## This module implements the new compilation cache.
import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types, import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
renderer, rodutils, std / sha1, idents, astalgo, magicsys renderer, rodutils, idents, astalgo, btrees, magicsys
## Todo: ## Todo:
## - Implement the 'import' replay logic so that the codegen runs over ## - Implement the 'import' replay logic so that the codegen runs over
@ -19,45 +19,44 @@ import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
## dependencies. ## dependencies.
## - Test multi methods. ## - Test multi methods.
## - Implement the limited VM support based on sets. ## - Implement the limited VM support based on sets.
## - Depencency computation should use signature hashes in order to ## - Depencency computation should use *signature* hashes in order to
## avoid recompiling dependent modules. ## avoid recompiling dependent modules.
template db(): DbConn = g.incr.db template db(): DbConn = g.incr.db
proc needsRecompile(g: ModuleGraph; fileIdx: int32; fullpath: string; proc needsRecompile(g: ModuleGraph; fileIdx: FileIndex; fullpath: string;
cycleCheck: var IntSet): bool = cycleCheck: var IntSet): bool =
let root = db.getRow(sql"select id, fullhash from filenames where fullpath = ?", let root = db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
fullpath) fullpath)
if root[0].len == 0: return true if root[0].len == 0: return true
if root[1] != hashFileCached(fileIdx, fullpath): if root[1] != hashFileCached(g.config, fileIdx, fullpath):
return true return true
# cycle detection: assume "not changed" is correct. # cycle detection: assume "not changed" is correct.
if cycleCheck.containsOrIncl(fileIdx): if cycleCheck.containsOrIncl(int fileIdx):
return false return false
# check dependencies (recursively): # check dependencies (recursively):
for row in db.fastRows(sql"select fullpath from filenames where id in (select dependency from deps where module = ?)", for row in db.fastRows(sql"select fullpath from filenames where id in (select dependency from deps where module = ?)",
root[0]): root[0]):
let dep = row[0] let dep = row[0]
if needsRecompile(g, dep.fileInfoIdx, dep, cycleCheck): if needsRecompile(g, g.config.fileInfoIdx(dep), dep, cycleCheck):
return true return true
return false return false
proc getModuleId*(g: ModuleGraph; fileIdx: int32; fullpath: string): int = proc getModuleId*(g: ModuleGraph; fileIdx: FileIndex; fullpath: string): int =
if gSymbolFiles != v2Sf: return getID() if g.config.symbolFiles != v2Sf: return getID()
let module = db.getRow( let module = g.incr.db.getRow(
sql"select id, fullHash from modules where fullpath = ?", fullpath) sql"select id, fullHash from modules where fullpath = ?", fullpath)
let currentFullhash = hashFileCached(fileIdx, fullpath) let currentFullhash = hashFileCached(g.config, fileIdx, fullpath)
if module[0].len == 0: if module[0].len == 0:
result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?, ?)", result = int db.insertID(sql"insert into modules(fullpath, interfHash, fullHash) values (?, ?, ?)",
fullpath, "", currentFullhash) fullpath, "", currentFullhash)
else: else:
result = parseInt(module[0]) result = parseInt(module[0])
if currentFullhash == module[1]: if currentFullhash == module[1]:
# not changed, so use the cached AST (even if it might be wrong # not changed, so use the cached AST:
# due to its dependencies):
doAssert(result != 0) doAssert(result != 0)
var cycleCheck = initIntSet() var cycleCheck = initIntSet()
if not needsRecompile(fileIdx, fullpath, cycleCheck): if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
return -result return -result
db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0]) db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
db.exec(sql"delete from deps where module = ?", module[0]) db.exec(sql"delete from deps where module = ?", module[0])
@ -66,20 +65,6 @@ proc getModuleId*(g: ModuleGraph; fileIdx: int32; fullpath: string): int =
db.exec(sql"delete from toplevelstmts where module = ?", module[0]) db.exec(sql"delete from toplevelstmts where module = ?", module[0])
db.exec(sql"delete from statics where module = ?", module[0]) db.exec(sql"delete from statics where module = ?", module[0])
type
TRodWriter = object
module: PSym
sstack: seq[PSym] # a stack of symbols to process
tstack: seq[PType] # a stack of types to process
tmarks, smarks: IntSet
forwardedSyms: seq[PSym]
PRodWriter = var TRodWriter
proc initRodWriter(module: PSym): TRodWriter =
result = TRodWriter(module: module, sstack: @[], tstack: @[],
tmarks: initIntSet(), smarks: initIntSet(), forwardedSyms: @[])
when false: when false:
proc getDefines(): string = proc getDefines(): string =
result = "" result = ""
@ -87,18 +72,17 @@ when false:
if result.len != 0: add(result, " ") if result.len != 0: add(result, " ")
add(result, d) add(result, d)
const proc pushType(w: var Writer, t: PType) =
rodNL = "\L"
proc pushType(w: PRodWriter, t: PType) =
if not containsOrIncl(w.tmarks, t.id): if not containsOrIncl(w.tmarks, t.id):
w.tstack.add(t) w.tstack.add(t)
proc pushSym(w: PRodWriter, s: PSym) = proc pushSym(w: var Writer, s: PSym) =
if not containsOrIncl(w.smarks, s.id): if not containsOrIncl(w.smarks, s.id):
w.sstack.add(s) w.sstack.add(s)
proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, template w: untyped = g.incr.w
proc encodeNode(g: ModuleGraph; fInfo: TLineInfo, n: PNode,
result: var string) = result: var string) =
if n == nil: if n == nil:
# nil nodes have to be stored too: # nil nodes have to be stored too:
@ -113,14 +97,14 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
result.add('?') result.add('?')
encodeVInt(n.info.col, result) encodeVInt(n.info.col, result)
result.add(',') result.add(',')
encodeVInt(n.info.line, result) encodeVInt(int n.info.line, result)
result.add(',') result.add(',')
encodeVInt(toDbFileId(n.info.fileIndex), result) encodeVInt(toDbFileId(g.incr, g.config, n.info.fileIndex), result)
elif fInfo.line != n.info.line: elif fInfo.line != n.info.line:
result.add('?') result.add('?')
encodeVInt(n.info.col, result) encodeVInt(n.info.col, result)
result.add(',') result.add(',')
encodeVInt(n.info.line, result) encodeVInt(int n.info.line, result)
elif fInfo.col != n.info.col: elif fInfo.col != n.info.col:
result.add('?') result.add('?')
encodeVInt(n.info.col, result) encodeVInt(n.info.col, result)
@ -156,10 +140,10 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
pushSym(w, n.sym) pushSym(w, n.sym)
else: else:
for i in countup(0, sonsLen(n) - 1): for i in countup(0, sonsLen(n) - 1):
encodeNode(w, n.info, n.sons[i], result) encodeNode(g, n.info, n.sons[i], result)
add(result, ')') add(result, ')')
proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = proc encodeLoc(g: ModuleGraph; loc: TLoc, result: var string) =
var oldLen = result.len var oldLen = result.len
result.add('<') result.add('<')
if loc.k != low(loc.k): encodeVInt(ord(loc.k), result) if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
@ -171,7 +155,7 @@ proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
encodeVInt(cast[int32](loc.flags), result) encodeVInt(cast[int32](loc.flags), result)
if loc.lode != nil: if loc.lode != nil:
add(result, '^') add(result, '^')
encodeNode(w, unknownLineInfo(), loc.lode, result) encodeNode(g, unknownLineInfo(), loc.lode, result)
#encodeVInt(cast[int32](loc.t.id), result) #encodeVInt(cast[int32](loc.t.id), result)
#pushType(w, loc.t) #pushType(w, loc.t)
if loc.r != nil: if loc.r != nil:
@ -183,13 +167,13 @@ proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
else: else:
add(result, '>') add(result, '>')
proc encodeType(w: PRodWriter, t: PType, result: var string) = proc encodeType(g: ModuleGraph, t: PType, result: var string) =
if t == nil: if t == nil:
# nil nodes have to be stored too: # nil nodes have to be stored too:
result.add("[]") result.add("[]")
return return
# we need no surrounding [] here because the type is in a line of its own # we need no surrounding [] here because the type is in a line of its own
if t.kind == tyForward: internalError("encodeType: tyForward") if t.kind == tyForward: internalError(g.config, "encodeType: tyForward")
# for the new rodfile viewer we use a preceding [ so that the data section # for the new rodfile viewer we use a preceding [ so that the data section
# can easily be disambiguated: # can easily be disambiguated:
add(result, '[') add(result, '[')
@ -197,7 +181,7 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
add(result, '+') add(result, '+')
encodeVInt(t.id, result) encodeVInt(t.id, result)
if t.n != nil: if t.n != nil:
encodeNode(w, w.module.info, t.n, result) encodeNode(g, unknownLineInfo(), t.n, result)
if t.flags != {}: if t.flags != {}:
add(result, '$') add(result, '$')
encodeVInt(cast[int32](t.flags), result) encodeVInt(cast[int32](t.flags), result)
@ -243,7 +227,7 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
add(result, '\20') add(result, '\20')
encodeVInt(s.id, result) encodeVInt(s.id, result)
pushSym(w, s) pushSym(w, s)
encodeLoc(w, t.loc, result) encodeLoc(g, t.loc, result)
for i in countup(0, sonsLen(t) - 1): for i in countup(0, sonsLen(t) - 1):
if t.sons[i] == nil: if t.sons[i] == nil:
add(result, "^()") add(result, "^()")
@ -252,15 +236,15 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
encodeVInt(t.sons[i].id, result) encodeVInt(t.sons[i].id, result)
pushType(w, t.sons[i]) pushType(w, t.sons[i])
proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = proc encodeLib(g: ModuleGraph, lib: PLib, info: TLineInfo, result: var string) =
add(result, '|') add(result, '|')
encodeVInt(ord(lib.kind), result) encodeVInt(ord(lib.kind), result)
add(result, '|') add(result, '|')
encodeStr($lib.name, result) encodeStr($lib.name, result)
add(result, '|') add(result, '|')
encodeNode(w, info, lib.path, result) encodeNode(g, info, lib.path, result)
proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation]; proc encodeInstantiations(g: ModuleGraph; s: seq[PInstantiation];
result: var string) = result: var string) =
for t in s: for t in s:
result.add('\15') result.add('\15')
@ -273,7 +257,7 @@ proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation];
result.add('\20') result.add('\20')
encodeVInt(t.compilesId, result) encodeVInt(t.compilesId, result)
proc encodeSym(w: PRodWriter, s: PSym, result: var string) = proc encodeSym(g: ModuleGraph, s: PSym, result: var string) =
if s == nil: if s == nil:
# nil nodes have to be stored too: # nil nodes have to be stored too:
result.add("{}") result.add("{}")
@ -291,9 +275,9 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
result.add('?') result.add('?')
if s.info.col != -1'i16: encodeVInt(s.info.col, result) if s.info.col != -1'i16: encodeVInt(s.info.col, result)
result.add(',') result.add(',')
if s.info.line != -1'i16: encodeVInt(s.info.line, result) encodeVInt(int s.info.line, result)
result.add(',') result.add(',')
encodeVInt(toDbFileId(s.info.fileIndex), result) encodeVInt(toDbFileId(g.incr, g.config, s.info.fileIndex), result)
if s.owner != nil: if s.owner != nil:
result.add('*') result.add('*')
encodeVInt(s.owner.id, result) encodeVInt(s.owner.id, result)
@ -312,11 +296,11 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
if s.offset != - 1: if s.offset != - 1:
result.add('`') result.add('`')
encodeVInt(s.offset, result) encodeVInt(s.offset, result)
encodeLoc(w, s.loc, result) encodeLoc(g, s.loc, result)
if s.annex != nil: encodeLib(w, s.annex, s.info, result) if s.annex != nil: encodeLib(g, s.annex, s.info, result)
if s.constraint != nil: if s.constraint != nil:
add(result, '#') add(result, '#')
encodeNode(w, unknownLineInfo(), s.constraint, result) encodeNode(g, unknownLineInfo(), s.constraint, result)
case s.kind case s.kind
of skType, skGenericParam: of skType, skGenericParam:
for t in s.typeInstCache: for t in s.typeInstCache:
@ -324,13 +308,13 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
encodeVInt(t.id, result) encodeVInt(t.id, result)
pushType(w, t) pushType(w, t)
of routineKinds: of routineKinds:
encodeInstantiations(w, s.procInstCache, result) encodeInstantiations(g, s.procInstCache, result)
if s.gcUnsafetyReason != nil: if s.gcUnsafetyReason != nil:
result.add('\16') result.add('\16')
encodeVInt(s.gcUnsafetyReason.id, result) encodeVInt(s.gcUnsafetyReason.id, result)
pushSym(w, s.gcUnsafetyReason) pushSym(w, s.gcUnsafetyReason)
of skModule, skPackage: of skModule, skPackage:
encodeInstantiations(w, s.usedGenerics, result) encodeInstantiations(g, s.usedGenerics, result)
# we don't serialize: # we don't serialize:
#tab*: TStrTable # interface table for modules #tab*: TStrTable # interface table for modules
of skLet, skVar, skField, skForVar: of skLet, skVar, skField, skForVar:
@ -348,47 +332,44 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
# we used to attempt to save space here by only storing a dummy AST if # we used to attempt to save space here by only storing a dummy AST if
# it is not necessary, but Nim's heavy compile-time evaluation features # it is not necessary, but Nim's heavy compile-time evaluation features
# make that unfeasible nowadays: # make that unfeasible nowadays:
encodeNode(w, s.info, s.ast, result) encodeNode(g, s.info, s.ast, result)
proc storeSym(w: PRodWriter; s: PSym) = proc storeSym(g: ModuleGraph; s: PSym) =
if sfForward in s.flags and s.kind != skModule: if sfForward in s.flags and s.kind != skModule:
w.forwardedSyms.add s w.forwardedSyms.add s
return return
var buf = newStringOfCap(160) var buf = newStringOfCap(160)
encodeSym(w, s, buf) encodeSym(g, s, buf)
# XXX only store the name for exported symbols in order to speed up lookup # XXX only store the name for exported symbols in order to speed up lookup
# times once we enable the skStub logic. # times once we enable the skStub logic.
db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)", db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)",
s.id, abs(w.module.id), s.name.s, buf, ord(sfExported in s.flags)) s.id, abs(getModule(s).id), s.name.s, buf, ord(sfExported in s.flags))
proc storeType(w: PRodWriter; t: PType) = proc storeType(g: ModuleGraph; t: PType) =
var buf = newStringOfCap(160) var buf = newStringOfCap(160)
encodeType(w, t, buf) encodeType(g, t, buf)
db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)", db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)",
t.id, abs(w.module.id), buf) t.id, abs(getModule(t.owner).id), buf)
var w = initRodWriter(nil)
proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) = proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
if gSymbolFiles != v2Sf: return if g.config.symbolFiles != v2Sf: return
w.module = module
var buf = newStringOfCap(160) var buf = newStringOfCap(160)
encodeNode(w, module.info, n, buf) encodeNode(g, module.info, n, buf)
db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)", db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)",
abs(module.id), module.offset, buf) abs(module.id), module.offset, buf)
inc module.offset inc module.offset
var i = 0 var i = 0
while true: while true:
if i > 10_000: if i > 10_000:
quit "loop never ends!" doAssert false, "loop never ends!"
if w.sstack.len > 0: if w.sstack.len > 0:
let s = w.sstack.pop() let s = w.sstack.pop()
when false: when false:
echo "popped ", s.name.s, " ", s.id echo "popped ", s.name.s, " ", s.id
storeSym(w, s) storeSym(g, s)
elif w.tstack.len > 0: elif w.tstack.len > 0:
let t = w.tstack.pop() let t = w.tstack.pop()
storeType(w, t) storeType(g, t)
when false: when false:
echo "popped type ", typeToString(t), " ", t.id echo "popped type ", typeToString(t), " ", t.id
else: else:
@ -396,57 +377,39 @@ proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
inc i inc i
proc storeRemaining*(g: ModuleGraph; module: PSym) = proc storeRemaining*(g: ModuleGraph; module: PSym) =
if gSymbolFiles != v2Sf: return if g.config.symbolFiles != v2Sf: return
w.module = module
for s in w.forwardedSyms: for s in w.forwardedSyms:
assert sfForward notin s.flags assert sfForward notin s.flags
storeSym(w, s) storeSym(g, s)
w.forwardedSyms.setLen 0 w.forwardedSyms.setLen 0
# ---------------- decoder ----------------------------------- # ---------------- decoder -----------------------------------
type type
TRodReader = object
module: PSym
#sstack: seq[(PSym, ptr PSym)] # a stack of symbols to process
#tstack: seq[(PType, ptr PType)] # a stack of types to process
#tmarks, smarks: IntSet
syms: Table[int, PSym] ## XXX make this more efficient
types: Table[int, PType]
cache: IdentCache
BlobReader = object BlobReader = object
s: string s: string
pos: int pos: int
PRodReader = var TRodReader
proc initRodReader(cache: IdentCache): TRodReader =
TRodReader(module: nil,
syms: initTable[int, PSym](), types: initTable[int, PType](),
cache: cache)
var gr = initRodReader(newIdentCache())
using using
r: PRodReader r: var Reader
b: var BlobReader b: var BlobReader
g: ModuleGraph
proc loadSym(r; id: int, info: TLineInfo): PSym proc loadSym(g; id: int, info: TLineInfo): PSym
proc loadType(r; id: int, info: TLineInfo): PType proc loadType(g; id: int, info: TLineInfo): PType
proc decodeLineInfo(r; b; info: var TLineInfo) = proc decodeLineInfo(g; b; info: var TLineInfo) =
if b.s[b.pos] == '?': if b.s[b.pos] == '?':
inc(b.pos) inc(b.pos)
if b.s[b.pos] == ',': info.col = -1'i16 if b.s[b.pos] == ',': info.col = -1'i16
else: info.col = int16(decodeVInt(b.s, b.pos)) else: info.col = int16(decodeVInt(b.s, b.pos))
if b.s[b.pos] == ',': if b.s[b.pos] == ',':
inc(b.pos) inc(b.pos)
if b.s[b.pos] == ',': info.line = -1'i16 if b.s[b.pos] == ',': info.line = 0'u16
else: info.line = int16(decodeVInt(b.s, b.pos)) else: info.line = uint16(decodeVInt(b.s, b.pos))
if b.s[b.pos] == ',': if b.s[b.pos] == ',':
inc(b.pos) inc(b.pos)
info.fileIndex = fromDbFileId(decodeVInt(b.s, b.pos)) info.fileIndex = fromDbFileId(g.incr, g.config, decodeVInt(b.s, b.pos))
proc skipNode(b) = proc skipNode(b) =
assert b.s[b.pos] == '(' assert b.s[b.pos] == '('
@ -462,7 +425,7 @@ proc skipNode(b) =
inc pos inc pos
b.pos = pos+1 # skip ')' b.pos = pos+1 # skip ')'
proc decodeNodeLazyBody(r; b; fInfo: TLineInfo, proc decodeNodeLazyBody(g; b; fInfo: TLineInfo,
belongsTo: PSym): PNode = belongsTo: PSym): PNode =
result = nil result = nil
if b.s[b.pos] == '(': if b.s[b.pos] == '(':
@ -471,14 +434,14 @@ proc decodeNodeLazyBody(r; b; fInfo: TLineInfo,
inc(b.pos) inc(b.pos)
return # nil node return # nil node
result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo) result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo)
decodeLineInfo(r, b, result.info) decodeLineInfo(g, b, result.info)
if b.s[b.pos] == '$': if b.s[b.pos] == '$':
inc(b.pos) inc(b.pos)
result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos))) result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos)))
if b.s[b.pos] == '^': if b.s[b.pos] == '^':
inc(b.pos) inc(b.pos)
var id = decodeVInt(b.s, b.pos) var id = decodeVInt(b.s, b.pos)
result.typ = loadType(r, id, result.info) result.typ = loadType(g, id, result.info)
case result.kind case result.kind
of nkCharLit..nkUInt64Lit: of nkCharLit..nkUInt64Lit:
if b.s[b.pos] == '!': if b.s[b.pos] == '!':
@ -499,16 +462,16 @@ proc decodeNodeLazyBody(r; b; fInfo: TLineInfo,
if b.s[b.pos] == '!': if b.s[b.pos] == '!':
inc(b.pos) inc(b.pos)
var fl = decodeStr(b.s, b.pos) var fl = decodeStr(b.s, b.pos)
result.ident = r.cache.getIdent(fl) result.ident = g.cache.getIdent(fl)
else: else:
internalError(result.info, "decodeNode: nkIdent") internalError(g.config, result.info, "decodeNode: nkIdent")
of nkSym: of nkSym:
if b.s[b.pos] == '!': if b.s[b.pos] == '!':
inc(b.pos) inc(b.pos)
var id = decodeVInt(b.s, b.pos) var id = decodeVInt(b.s, b.pos)
result.sym = loadSym(r, id, result.info) result.sym = loadSym(g, id, result.info)
else: else:
internalError(result.info, "decodeNode: nkSym") internalError(g.config, result.info, "decodeNode: nkSym")
else: else:
var i = 0 var i = 0
while b.s[b.pos] != ')': while b.s[b.pos] != ')':
@ -519,17 +482,17 @@ proc decodeNodeLazyBody(r; b; fInfo: TLineInfo,
skipNode(b) skipNode(b)
else: else:
discard discard
addSonNilAllowed(result, decodeNodeLazyBody(r, b, result.info, nil)) addSonNilAllowed(result, decodeNodeLazyBody(g, b, result.info, nil))
inc i inc i
if b.s[b.pos] == ')': inc(b.pos) if b.s[b.pos] == ')': inc(b.pos)
else: internalError(result.info, "decodeNode: ')' missing") else: internalError(g.config, result.info, "decodeNode: ')' missing")
else: else:
internalError(fInfo, "decodeNode: '(' missing " & $b.pos) internalError(g.config, fInfo, "decodeNode: '(' missing " & $b.pos)
proc decodeNode(r; b; fInfo: TLineInfo): PNode = proc decodeNode(g; b; fInfo: TLineInfo): PNode =
result = decodeNodeLazyBody(r, b, fInfo, nil) result = decodeNodeLazyBody(g, b, fInfo, nil)
proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) = proc decodeLoc(g; b; loc: var TLoc, info: TLineInfo) =
if b.s[b.pos] == '<': if b.s[b.pos] == '<':
inc(b.pos) inc(b.pos)
if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}: if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}:
@ -548,7 +511,7 @@ proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) =
loc.flags = {} loc.flags = {}
if b.s[b.pos] == '^': if b.s[b.pos] == '^':
inc(b.pos) inc(b.pos)
loc.lode = decodeNode(r, b, info) loc.lode = decodeNode(g, b, info)
# rrGetType(b, decodeVInt(b.s, b.pos), info) # rrGetType(b, decodeVInt(b.s, b.pos), info)
else: else:
loc.lode = nil loc.lode = nil
@ -558,19 +521,19 @@ proc decodeLoc(r; b; loc: var TLoc, info: TLineInfo) =
else: else:
loc.r = nil loc.r = nil
if b.s[b.pos] == '>': inc(b.pos) if b.s[b.pos] == '>': inc(b.pos)
else: internalError(info, "decodeLoc " & b.s[b.pos]) else: internalError(g.config, info, "decodeLoc " & b.s[b.pos])
proc loadBlob(query: SqlQuery; id: int): BlobReader = proc loadBlob(g; query: SqlQuery; id: int): BlobReader =
let blob = db.getValue(query, id) let blob = db.getValue(query, id)
if blob.len == 0: if blob.len == 0:
internalError("symbolfiles: cannot find ID " & $ id) internalError(g.config, "symbolfiles: cannot find ID " & $ id)
result = BlobReader(pos: 0) result = BlobReader(pos: 0)
shallowCopy(result.s, blob) shallowCopy(result.s, blob)
proc loadType(r; id: int; info: TLineInfo): PType = proc loadType(g; id: int; info: TLineInfo): PType =
result = r.types.getOrDefault(id) result = g.incr.r.types.getOrDefault(id)
if result != nil: return result if result != nil: return result
var b = loadBlob(sql"select data from types where nimid = ?", id) var b = loadBlob(g, sql"select data from types where nimid = ?", id)
if b.s[b.pos] == '[': if b.s[b.pos] == '[':
inc(b.pos) inc(b.pos)
@ -585,10 +548,10 @@ proc loadType(r; id: int; info: TLineInfo): PType =
setId(result.id) setId(result.id)
#if debugIds: registerID(result) #if debugIds: registerID(result)
else: else:
internalError(info, "decodeType: no id") internalError(g.config, info, "decodeType: no id")
# here this also avoids endless recursion for recursive type # here this also avoids endless recursion for recursive type
r.types[result.id] = result g.incr.r.types.add(result.id, result)
if b.s[b.pos] == '(': result.n = decodeNode(r, b, unknownLineInfo()) if b.s[b.pos] == '(': result.n = decodeNode(g, b, unknownLineInfo())
if b.s[b.pos] == '$': if b.s[b.pos] == '$':
inc(b.pos) inc(b.pos)
result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos))) result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos)))
@ -597,10 +560,10 @@ proc loadType(r; id: int; info: TLineInfo): PType =
result.callConv = TCallingConvention(decodeVInt(b.s, b.pos)) result.callConv = TCallingConvention(decodeVInt(b.s, b.pos))
if b.s[b.pos] == '*': if b.s[b.pos] == '*':
inc(b.pos) inc(b.pos)
result.owner = loadSym(r, decodeVInt(b.s, b.pos), info) result.owner = loadSym(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '&': if b.s[b.pos] == '&':
inc(b.pos) inc(b.pos)
result.sym = loadSym(r, decodeVInt(b.s, b.pos), info) result.sym = loadSym(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '/': if b.s[b.pos] == '/':
inc(b.pos) inc(b.pos)
result.size = decodeVInt(b.s, b.pos) result.size = decodeVInt(b.s, b.pos)
@ -620,65 +583,65 @@ proc loadType(r; id: int; info: TLineInfo): PType =
if b.s[b.pos] == '\15': if b.s[b.pos] == '\15':
inc(b.pos) inc(b.pos)
result.destructor = loadSym(r, decodeVInt(b.s, b.pos), info) result.destructor = loadSym(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '\16': if b.s[b.pos] == '\16':
inc(b.pos) inc(b.pos)
result.deepCopy = loadSym(r, decodeVInt(b.s, b.pos), info) result.deepCopy = loadSym(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '\17': if b.s[b.pos] == '\17':
inc(b.pos) inc(b.pos)
result.assignment = loadSym(r, decodeVInt(b.s, b.pos), info) result.assignment = loadSym(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '\18': if b.s[b.pos] == '\18':
inc(b.pos) inc(b.pos)
result.sink = loadSym(r, decodeVInt(b.s, b.pos), info) result.sink = loadSym(g, decodeVInt(b.s, b.pos), info)
while b.s[b.pos] == '\19': while b.s[b.pos] == '\19':
inc(b.pos) inc(b.pos)
let x = decodeVInt(b.s, b.pos) let x = decodeVInt(b.s, b.pos)
doAssert b.s[b.pos] == '\20' doAssert b.s[b.pos] == '\20'
inc(b.pos) inc(b.pos)
let y = loadSym(r, decodeVInt(b.s, b.pos), info) let y = loadSym(g, decodeVInt(b.s, b.pos), info)
result.methods.safeAdd((x, y)) result.methods.safeAdd((x, y))
decodeLoc(r, b, result.loc, info) decodeLoc(g, b, result.loc, info)
while b.s[b.pos] == '^': while b.s[b.pos] == '^':
inc(b.pos) inc(b.pos)
if b.s[b.pos] == '(': if b.s[b.pos] == '(':
inc(b.pos) inc(b.pos)
if b.s[b.pos] == ')': inc(b.pos) if b.s[b.pos] == ')': inc(b.pos)
else: internalError(info, "decodeType ^(" & b.s[b.pos]) else: internalError(g.config, info, "decodeType ^(" & b.s[b.pos])
rawAddSon(result, nil) rawAddSon(result, nil)
else: else:
var d = decodeVInt(b.s, b.pos) let d = decodeVInt(b.s, b.pos)
rawAddSon(result, loadType(r, d, info)) rawAddSon(result, loadType(g, d, info))
proc decodeLib(r; b; info: TLineInfo): PLib = proc decodeLib(g; b; info: TLineInfo): PLib =
result = nil result = nil
if b.s[b.pos] == '|': if b.s[b.pos] == '|':
new(result) new(result)
inc(b.pos) inc(b.pos)
result.kind = TLibKind(decodeVInt(b.s, b.pos)) result.kind = TLibKind(decodeVInt(b.s, b.pos))
if b.s[b.pos] != '|': internalError("decodeLib: 1") if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 1")
inc(b.pos) inc(b.pos)
result.name = rope(decodeStr(b.s, b.pos)) result.name = rope(decodeStr(b.s, b.pos))
if b.s[b.pos] != '|': internalError("decodeLib: 2") if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 2")
inc(b.pos) inc(b.pos)
result.path = decodeNode(r, b, info) result.path = decodeNode(g, b, info)
proc decodeInstantiations(r; b; info: TLineInfo; proc decodeInstantiations(g; b; info: TLineInfo;
s: var seq[PInstantiation]) = s: var seq[PInstantiation]) =
while b.s[b.pos] == '\15': while b.s[b.pos] == '\15':
inc(b.pos) inc(b.pos)
var ii: PInstantiation var ii: PInstantiation
new ii new ii
ii.sym = loadSym(r, decodeVInt(b.s, b.pos), info) ii.sym = loadSym(g, decodeVInt(b.s, b.pos), info)
ii.concreteTypes = @[] ii.concreteTypes = @[]
while b.s[b.pos] == '\17': while b.s[b.pos] == '\17':
inc(b.pos) inc(b.pos)
ii.concreteTypes.add loadType(r, decodeVInt(b.s, b.pos), info) ii.concreteTypes.add loadType(g, decodeVInt(b.s, b.pos), info)
if b.s[b.pos] == '\20': if b.s[b.pos] == '\20':
inc(b.pos) inc(b.pos)
ii.compilesId = decodeVInt(b.s, b.pos) ii.compilesId = decodeVInt(b.s, b.pos)
s.safeAdd ii s.safeAdd ii
proc loadSymFromBlob(r; b; info: TLineInfo): PSym = proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
if b.s[b.pos] == '{': if b.s[b.pos] == '{':
inc(b.pos) inc(b.pos)
if b.s[b.pos] == '}': if b.s[b.pos] == '}':
@ -691,26 +654,26 @@ proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
id = decodeVInt(b.s, b.pos) id = decodeVInt(b.s, b.pos)
setId(id) setId(id)
else: else:
internalError(info, "decodeSym: no id") internalError(g.config, info, "decodeSym: no id")
var ident: PIdent var ident: PIdent
if b.s[b.pos] == '&': if b.s[b.pos] == '&':
inc(b.pos) inc(b.pos)
ident = r.cache.getIdent(decodeStr(b.s, b.pos)) ident = g.cache.getIdent(decodeStr(b.s, b.pos))
else: else:
internalError(info, "decodeSym: no ident") internalError(g.config, info, "decodeSym: no ident")
#echo "decoding: {", ident.s #echo "decoding: {", ident.s
new(result) new(result)
result.id = id result.id = id
result.kind = k result.kind = k
result.name = ident # read the rest of the symbol description: result.name = ident # read the rest of the symbol description:
r.syms[result.id] = result g.incr.r.syms.add(result.id, result)
if b.s[b.pos] == '^': if b.s[b.pos] == '^':
inc(b.pos) inc(b.pos)
result.typ = loadType(r, decodeVInt(b.s, b.pos), info) result.typ = loadType(g, decodeVInt(b.s, b.pos), info)
decodeLineInfo(r, b, result.info) decodeLineInfo(g, b, result.info)
if b.s[b.pos] == '*': if b.s[b.pos] == '*':
inc(b.pos) inc(b.pos)
result.owner = loadSym(r, decodeVInt(b.s, b.pos), result.info) result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info)
if b.s[b.pos] == '$': if b.s[b.pos] == '$':
inc(b.pos) inc(b.pos)
result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos))) result.flags = cast[TSymFlags](int32(decodeVInt(b.s, b.pos)))
@ -720,8 +683,6 @@ proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
if b.s[b.pos] == '!': if b.s[b.pos] == '!':
inc(b.pos) inc(b.pos)
result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos))) result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos)))
else:
result.options = r.module.options
if b.s[b.pos] == '%': if b.s[b.pos] == '%':
inc(b.pos) inc(b.pos)
result.position = decodeVInt(b.s, b.pos) result.position = decodeVInt(b.s, b.pos)
@ -729,28 +690,28 @@ proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
inc(b.pos) inc(b.pos)
result.offset = decodeVInt(b.s, b.pos) result.offset = decodeVInt(b.s, b.pos)
else: else:
result.offset = - 1 result.offset = -1
decodeLoc(r, b, result.loc, result.info) decodeLoc(g, b, result.loc, result.info)
result.annex = decodeLib(r, b, info) result.annex = decodeLib(g, b, info)
if b.s[b.pos] == '#': if b.s[b.pos] == '#':
inc(b.pos) inc(b.pos)
result.constraint = decodeNode(r, b, unknownLineInfo()) result.constraint = decodeNode(g, b, unknownLineInfo())
case result.kind case result.kind
of skType, skGenericParam: of skType, skGenericParam:
while b.s[b.pos] == '\14': while b.s[b.pos] == '\14':
inc(b.pos) inc(b.pos)
result.typeInstCache.safeAdd loadType(r, decodeVInt(b.s, b.pos), result.info) result.typeInstCache.safeAdd loadType(g, decodeVInt(b.s, b.pos), result.info)
of routineKinds: of routineKinds:
decodeInstantiations(r, b, result.info, result.procInstCache) decodeInstantiations(g, b, result.info, result.procInstCache)
if b.s[b.pos] == '\16': if b.s[b.pos] == '\16':
inc(b.pos) inc(b.pos)
result.gcUnsafetyReason = loadSym(r, decodeVInt(b.s, b.pos), result.info) result.gcUnsafetyReason = loadSym(g, decodeVInt(b.s, b.pos), result.info)
of skModule, skPackage: of skModule, skPackage:
decodeInstantiations(r, b, result.info, result.usedGenerics) decodeInstantiations(g, b, result.info, result.usedGenerics)
of skLet, skVar, skField, skForVar: of skLet, skVar, skField, skForVar:
if b.s[b.pos] == '\18': if b.s[b.pos] == '\18':
inc(b.pos) inc(b.pos)
result.guard = loadSym(r, decodeVInt(b.s, b.pos), result.info) result.guard = loadSym(g, decodeVInt(b.s, b.pos), result.info)
if b.s[b.pos] == '\19': if b.s[b.pos] == '\19':
inc(b.pos) inc(b.pos)
result.bitsize = decodeVInt(b.s, b.pos).int16 result.bitsize = decodeVInt(b.s, b.pos).int16
@ -760,37 +721,37 @@ proc loadSymFromBlob(r; b; info: TLineInfo): PSym =
#if result.kind in routineKinds: #if result.kind in routineKinds:
# result.ast = decodeNodeLazyBody(b, result.info, result) # result.ast = decodeNodeLazyBody(b, result.info, result)
#else: #else:
result.ast = decodeNode(r, b, result.info) result.ast = decodeNode(g, b, result.info)
if sfCompilerProc in result.flags: if sfCompilerProc in result.flags:
registerCompilerProc(result) registerCompilerProc(g, result)
#echo "loading ", result.name.s #echo "loading ", result.name.s
proc loadSym(r; id: int; info: TLineInfo): PSym = proc loadSym(g; id: int; info: TLineInfo): PSym =
result = r.syms.getOrDefault(id) result = g.incr.r.syms.getOrDefault(id)
if result != nil: return result if result != nil: return result
var b = loadBlob(sql"select data from syms where nimid = ?", id) var b = loadBlob(g, sql"select data from syms where nimid = ?", id)
result = loadSymFromBlob(r, b, info) result = loadSymFromBlob(g, b, info)
doAssert id == result.id, "symbol ID is not consistent!" doAssert id == result.id, "symbol ID is not consistent!"
proc loadModuleSymTab(r; module: PSym) = proc loadModuleSymTab(g; module: PSym) =
## goal: fill module.tab ## goal: fill module.tab
gr.syms[module.id] = module g.incr.r.syms.add(module.id, module)
for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)): for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)):
let id = parseInt(row[0]) let id = parseInt(row[0])
var s = r.syms.getOrDefault(id) var s = g.incr.r.syms.getOrDefault(id)
if s == nil: if s == nil:
var b = BlobReader(pos: 0) var b = BlobReader(pos: 0)
shallowCopy(b.s, row[1]) shallowCopy(b.s, row[1])
s = loadSymFromBlob(r, b, module.info) s = loadSymFromBlob(g, b, module.info)
assert s != nil assert s != nil
strTableAdd(module.tab, s) strTableAdd(module.tab, s)
if sfSystemModule in module.flags: if sfSystemModule in module.flags:
magicsys.systemModule = module g.systemModule = module
proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode = proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode =
assert gSymbolFiles == v2Sf assert g.config.symbolFiles == v2Sf
if index == 0: if index == 0:
loadModuleSymTab(gr, module) loadModuleSymTab(g, module)
#index = parseInt db.getValue( #index = parseInt db.getValue(
# sql"select min(id) from toplevelstmts where module = ?", abs module.id) # sql"select min(id) from toplevelstmts where module = ?", abs module.id)
var b = BlobReader(pos: 0) var b = BlobReader(pos: 0)
@ -799,16 +760,15 @@ proc loadNode*(g: ModuleGraph; module: PSym; index: int): PNode =
if b.s.len == 0: if b.s.len == 0:
db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId) db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
return nil # end marker return nil # end marker
gr.module = module result = decodeNode(g, b, module.info)
result = decodeNode(gr, b, module.info)
proc setupModuleCache*(g: ModuleGraph) = proc setupModuleCache*(g: ModuleGraph) =
if gSymbolFiles != v2Sf: return if g.config.symbolFiles != v2Sf: return
let dbfile = getNimcacheDir(g.config) / "rodfiles.db" let dbfile = getNimcacheDir(g.config) / "rodfiles.db"
if not fileExists(dbfile): if not fileExists(dbfile):
db = open(connection=dbfile, user="nim", password="", db = open(connection=dbfile, user="nim", password="",
database="nim") database="nim")
createDb() createDb(db)
else: else:
db = open(connection=dbfile, user="nim", password="", db = open(connection=dbfile, user="nim", password="",
database="nim") database="nim")