big steps torwards an efficient, simple IC implementation (#16543)
* reworked ID handling * the packed AST now has its own ID mechanism * basic serialization code works * extract rodfiles to its own module * rodfiles: store and compare configs * rodfiles: store dependencies * store config at the end * precise dependency tracking * dependency tracking for rodfiles * completed loading of PSym, PType, etc * removed dead code * bugfix: do not realloc seqs when taking addr into an element * make IC opt-in for now * makes tcompilerapi green again * final cleanups Co-authored-by: Andy Davidoff <github@andy.disruptek.com>
This commit is contained in:
Andreas Rumpf
GitHub
parent
0d0e43469f
commit
73a8b950cb
57 changed files with 1429 additions and 1717 deletions
|
|
@ -1,7 +1,7 @@
|
|||
## A BiTable is a table that can be seen as an optimized pair
|
||||
## of (Table[LitId, Val], Table[Val, LitId]).
|
||||
|
||||
import hashes
|
||||
import hashes, rodfiles
|
||||
|
||||
type
|
||||
LitId* = distinct uint32
|
||||
|
|
@ -30,7 +30,9 @@ proc mustRehash(length, counter: int): bool {.inline.} =
|
|||
result = (length * 2 < counter * 3) or (length - counter < 4)
|
||||
|
||||
const
|
||||
idStart = 256 # Ids do not start with 0 but with this value. The IR needs it.
|
||||
idStart = 256 ##
|
||||
## Ids do not start with 0 but with this value. The IR needs it.
|
||||
## TODO: explain why
|
||||
|
||||
template idToIdx(x: LitId): int = x.int - idStart
|
||||
|
||||
|
|
@ -94,6 +96,21 @@ proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
|
|||
assert idx < t.vals.len
|
||||
result = t.vals[idx]
|
||||
|
||||
proc hash*[T](t: BiTable[T]): Hash =
|
||||
## as the keys are hashes of the values, we simply use them instead
|
||||
var h: Hash = 0
|
||||
for i, n in pairs t.keys:
|
||||
h = h !& hash((i, n))
|
||||
result = !$h
|
||||
|
||||
proc store*[T](f: var RodFile; t: BiTable[T]) =
|
||||
storeSeq(f, t.vals)
|
||||
storeSeq(f, t.keys)
|
||||
|
||||
proc load*[T](f: var RodFile; t: var BiTable[T]) =
|
||||
loadSeq(f, t.vals)
|
||||
loadSeq(f, t.keys)
|
||||
|
||||
when isMainModule:
|
||||
|
||||
var t: BiTable[string]
|
||||
|
|
@ -113,7 +130,35 @@ when isMainModule:
|
|||
|
||||
for i in 0 ..< 100_000:
|
||||
assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4
|
||||
echo "begin"
|
||||
echo t.vals.len
|
||||
|
||||
echo t.vals[0]
|
||||
echo t.vals[1004]
|
||||
|
||||
echo "middle"
|
||||
|
||||
var tf: BiTable[float]
|
||||
|
||||
discard tf.getOrIncl(0.4)
|
||||
discard tf.getOrIncl(16.4)
|
||||
discard tf.getOrIncl(32.4)
|
||||
echo getKeyId(tf, 32.4)
|
||||
|
||||
var f2 = open("testblah.bin", fmWrite)
|
||||
echo store(f2, tf)
|
||||
f2.close
|
||||
|
||||
var f1 = open("testblah.bin", fmRead)
|
||||
|
||||
var t2: BiTable[float]
|
||||
|
||||
echo f1.load(t2)
|
||||
echo t2.vals.len
|
||||
|
||||
echo getKeyId(t2, 32.4)
|
||||
|
||||
echo "end"
|
||||
|
||||
|
||||
f1.close
|
||||
|
|
|
|||
|
|
@ -29,14 +29,13 @@ mechanism needs to be implemented that we could get wrong. ModuleIds
|
|||
are rod-file specific too.
|
||||
|
||||
|
||||
Configuration setup changes
|
||||
---------------------------
|
||||
|
||||
For a MVP these are not detected. Later the configuration will be
|
||||
stored in every `.rod` file.
|
||||
|
||||
|
||||
Global state
|
||||
------------
|
||||
|
||||
Global persistent state will be kept in a project specific `.rod` file.
|
||||
|
||||
Rod File Format
|
||||
---------------
|
||||
|
||||
It's a simple binary file format. `rodfiles.nim` contains some details.
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
#
|
||||
#
|
||||
# The Nim Compiler
|
||||
# (c) Copyright 2020 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
import std / [hashes, tables]
|
||||
import bitabs
|
||||
import ".." / [ast, lineinfos, options, pathutils]
|
||||
|
|
@ -12,9 +12,9 @@
|
|||
## use this representation directly in all the transformations,
|
||||
## it is superior.
|
||||
|
||||
import std / [hashes, tables]
|
||||
import std / [hashes, tables, strtabs, md5]
|
||||
import bitabs
|
||||
import ".." / [ast, lineinfos, options, pathutils]
|
||||
import ".." / [ast, options]
|
||||
|
||||
const
|
||||
localNamePos* = 0
|
||||
|
|
@ -39,16 +39,29 @@ const
|
|||
routineBodyPos* = 7
|
||||
|
||||
const
|
||||
nkModuleRef = nkNone # pair of (ModuleId, SymId)
|
||||
nkModuleRef* = nkNone # pair of (ModuleId, SymId)
|
||||
|
||||
type
|
||||
SymId* = distinct int32
|
||||
TypeId* = distinct int32
|
||||
ModuleId* = distinct int32
|
||||
NodePos* = distinct int
|
||||
|
||||
NodeId* = distinct int32
|
||||
|
||||
PackedItemId* = object
|
||||
module*: LitId # 0 if it's this module
|
||||
item*: int32 # same as the in-memory representation
|
||||
|
||||
TypeId* = PackedItemId
|
||||
|
||||
const
|
||||
nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32)
|
||||
nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
|
||||
|
||||
const
|
||||
emptyNodeId* = NodeId(-1)
|
||||
|
||||
type
|
||||
PackedLineInfo* = object
|
||||
line*: uint16
|
||||
col*: int16
|
||||
|
|
@ -64,13 +77,13 @@ type
|
|||
PackedSym* = object
|
||||
kind*: TSymKind
|
||||
name*: LitId
|
||||
typeId*: TypeId
|
||||
typ*: TypeId
|
||||
flags*: TSymFlags
|
||||
magic*: TMagic
|
||||
info*: PackedLineInfo
|
||||
ast*: NodePos
|
||||
owner*: ItemId
|
||||
guard*: ItemId
|
||||
ast*: NodeId
|
||||
owner*: PackedItemId
|
||||
guard*: PackedItemId
|
||||
bitsize*: int
|
||||
alignment*: int # for alignment
|
||||
options*: TOptions
|
||||
|
|
@ -84,25 +97,25 @@ type
|
|||
|
||||
PackedType* = object
|
||||
kind*: TTypeKind
|
||||
nodekind*: TNodeKind
|
||||
callConv*: TCallingConvention
|
||||
#nodekind*: TNodeKind
|
||||
flags*: TTypeFlags
|
||||
types*: int32
|
||||
nodes*: int32
|
||||
methods*: int32
|
||||
nodeflags*: TNodeFlags
|
||||
info*: PackedLineInfo
|
||||
sym*: ItemId
|
||||
owner*: ItemId
|
||||
attachedOps*: array[TTypeAttachedOp, ItemId]
|
||||
types*: seq[TypeId]
|
||||
n*: NodeId
|
||||
methods*: seq[(int, PackedItemId)]
|
||||
#nodeflags*: TNodeFlags
|
||||
sym*: PackedItemId
|
||||
owner*: PackedItemId
|
||||
attachedOps*: array[TTypeAttachedOp, PackedItemId]
|
||||
size*: BiggestInt
|
||||
align*: int16
|
||||
paddingAtEnd*: int16
|
||||
lockLevel*: TLockLevel # lock level as required for deadlock checking
|
||||
# not serialized: loc*: TLoc because it is backend-specific
|
||||
typeInst*: TypeId
|
||||
nonUniqueId*: ItemId
|
||||
nonUniqueId*: int32
|
||||
|
||||
Node* = object # 20 bytes
|
||||
PackedNode* = object # 20 bytes
|
||||
kind*: TNodeKind
|
||||
flags*: TNodeFlags
|
||||
operand*: int32 # for kind in {nkSym, nkSymDef}: SymId
|
||||
|
|
@ -115,72 +128,73 @@ type
|
|||
ModulePhase* = enum
|
||||
preLookup, lookedUpTopLevelStmts
|
||||
|
||||
Module* = object
|
||||
GenericKey* = object
|
||||
module*: int32
|
||||
name*: string
|
||||
file*: AbsoluteFile
|
||||
ast*: PackedTree
|
||||
phase*: ModulePhase
|
||||
iface*: Table[string, seq[SymId]] # 'seq' because of overloading
|
||||
types*: seq[MD5Digest] # is this a joke?
|
||||
|
||||
Program* = ref object
|
||||
modules*: seq[Module]
|
||||
PackedTree* = object ## usually represents a full Nim module
|
||||
nodes*: seq[PackedNode]
|
||||
#sh*: Shared
|
||||
|
||||
Shared* = ref object # shared between different versions of 'Module'.
|
||||
# (though there is always exactly one valid
|
||||
# version of a module)
|
||||
syms*: seq[PackedSym]
|
||||
types*: seq[seq[Node]]
|
||||
types*: seq[PackedType]
|
||||
strings*: BiTable[string] # we could share these between modules.
|
||||
integers*: BiTable[BiggestInt]
|
||||
floats*: BiTable[BiggestFloat]
|
||||
config*: ConfigRef
|
||||
#thisModule*: ModuleId
|
||||
#program*: Program
|
||||
#config*: ConfigRef
|
||||
|
||||
PackedTree* = object ## usually represents a full Nim module
|
||||
nodes*: seq[Node]
|
||||
toPosition*: Table[SymId, NodePos]
|
||||
sh*: Shared
|
||||
proc hash*(key: GenericKey): Hash =
|
||||
var h: Hash = 0
|
||||
h = h !& hash(key.module)
|
||||
h = h !& hash(key.name)
|
||||
h = h !& hash(key.types)
|
||||
result = !$h
|
||||
|
||||
proc `==`*(a, b: SymId): bool {.borrow.}
|
||||
proc hash*(a: SymId): Hash {.borrow.}
|
||||
|
||||
proc `==`*(a, b: NodePos): bool {.borrow.}
|
||||
proc `==`*(a, b: TypeId): bool {.borrow.}
|
||||
proc `==`*(a, b: ModuleId): bool {.borrow.}
|
||||
|
||||
proc declareSym*(tree: var PackedTree; kind: TSymKind;
|
||||
name: LitId; info: PackedLineInfo): SymId =
|
||||
result = SymId(tree.sh.syms.len)
|
||||
tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
|
||||
#proc `==`*(a, b: TypeId): bool {.borrow.}
|
||||
proc `==`*(a, b: NodeId): bool {.borrow.}
|
||||
|
||||
proc newTreeFrom*(old: PackedTree): PackedTree =
|
||||
result.nodes = @[]
|
||||
result.sh = old.sh
|
||||
when false: result.sh = old.sh
|
||||
|
||||
proc litIdFromName*(tree: PackedTree; name: string): LitId =
|
||||
result = tree.sh.strings.getOrIncl(name)
|
||||
when false:
|
||||
proc declareSym*(tree: var PackedTree; kind: TSymKind;
|
||||
name: LitId; info: PackedLineInfo): SymId =
|
||||
result = SymId(tree.sh.syms.len)
|
||||
tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
|
||||
|
||||
proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: kind, operand: int32 getOrIncl(tree.sh.strings, token), info: info)
|
||||
proc litIdFromName*(tree: PackedTree; name: string): LitId =
|
||||
result = tree.sh.strings.getOrIncl(name)
|
||||
|
||||
proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: kind, operand: 0, info: info)
|
||||
proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
|
||||
tree.nodes.add PackedNode(kind: kind, info: info,
|
||||
operand: int32 getOrIncl(tree.sh.strings, token))
|
||||
|
||||
proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
|
||||
tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
|
||||
|
||||
proc throwAwayLastNode*(tree: var PackedTree) =
|
||||
tree.nodes.setLen(tree.nodes.len-1)
|
||||
|
||||
proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: nkIdent, operand: int32(s), info: info)
|
||||
tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
|
||||
|
||||
proc addSym*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
|
||||
proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
|
||||
tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
|
||||
|
||||
proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: nkInt32Lit, operand: int32(s), info: info)
|
||||
tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
|
||||
|
||||
proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
|
||||
tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
|
||||
tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
|
||||
|
||||
proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
|
||||
|
||||
|
|
@ -194,11 +208,12 @@ proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
|
|||
for i in 0..<L:
|
||||
dest.nodes[d+i] = tree.nodes[pos+i]
|
||||
|
||||
proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
|
||||
result = SymId(dest.sh.syms.len)
|
||||
assert int(s) < tree.sh.syms.len
|
||||
let oldSym = tree.sh.syms[s.int]
|
||||
dest.sh.syms.add oldSym
|
||||
when false:
|
||||
proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
|
||||
result = SymId(dest.sh.syms.len)
|
||||
assert int(s) < tree.sh.syms.len
|
||||
let oldSym = tree.sh.syms[s.int]
|
||||
dest.sh.syms.add oldSym
|
||||
|
||||
type
|
||||
PatchPos = distinct int
|
||||
|
|
@ -206,11 +221,12 @@ type
|
|||
when false:
|
||||
proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
|
||||
result = PatchPos tree.nodes.len
|
||||
tree.nodes.add Node(kind: kind, operand: 0, info: info)
|
||||
tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
|
||||
|
||||
proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: TypeId; info: PackedLineInfo): PatchPos =
|
||||
result = PatchPos tree.nodes.len
|
||||
tree.nodes.add Node(kind: kind, flags: flags, operand: 0, typeId: typeId, info: info)
|
||||
tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
|
||||
typeId: typeId)
|
||||
|
||||
proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
|
||||
result = PatchPos dest.nodes.len
|
||||
|
|
@ -224,7 +240,8 @@ proc patch*(tree: var PackedTree; pos: PatchPos) =
|
|||
|
||||
proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
|
||||
|
||||
proc `[]`*(tree: PackedTree; i: int): lent Node {.inline.} = tree.nodes[i]
|
||||
proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
|
||||
tree.nodes[i]
|
||||
|
||||
proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
|
||||
if tree.nodes[pos].kind > nkNilLit:
|
||||
|
|
@ -247,7 +264,8 @@ iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
|
|||
for x in sonsReadonly(tree, n): yield x
|
||||
patch dest, patchPos
|
||||
|
||||
iterator isons*(dest: var PackedTree; tree: PackedTree; n: NodePos): (int, NodePos) =
|
||||
iterator isons*(dest: var PackedTree; tree: PackedTree;
|
||||
n: NodePos): (int, NodePos) =
|
||||
var i = 0
|
||||
for ch0 in sons(dest, tree, n):
|
||||
yield (i, ch0)
|
||||
|
|
@ -301,10 +319,19 @@ proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
|
|||
if count >= x: return true
|
||||
return false
|
||||
|
||||
proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
|
||||
proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind
|
||||
proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = LitId tree.nodes[n.int].operand
|
||||
proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info
|
||||
proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
|
||||
NodePos(n.int+1)
|
||||
proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
|
||||
tree.nodes[n.int].kind
|
||||
proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
|
||||
LitId tree.nodes[n.int].operand
|
||||
proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
|
||||
tree.nodes[n.int].info
|
||||
|
||||
template typ*(n: NodePos): PackedItemId =
|
||||
tree.nodes[n.int].typeId
|
||||
template flags*(n: NodePos): TNodeFlags =
|
||||
tree.nodes[n.int].flags
|
||||
|
||||
proc span(tree: PackedTree; pos: int): int {.inline.} =
|
||||
if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
|
||||
|
|
@ -330,7 +357,9 @@ proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
|
|||
inc count
|
||||
assert false, "node has no i-th child"
|
||||
|
||||
proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = tree.sh.strings[lit]
|
||||
when false:
|
||||
proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} =
|
||||
tree.sh.strings[lit]
|
||||
|
||||
template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
|
||||
template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
|
||||
|
|
@ -340,29 +369,30 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
|
|||
|
||||
proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
|
||||
|
||||
proc strLit*(tree: PackedTree; n: NodePos): lent string =
|
||||
assert n.kind == nkStrLit
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
when false:
|
||||
proc strLit*(tree: PackedTree; n: NodePos): lent string =
|
||||
assert n.kind == nkStrLit
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
|
||||
proc strVal*(tree: PackedTree; n: NodePos): string =
|
||||
assert n.kind == nkStrLit
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
#result = cookedStrLit(raw)
|
||||
proc strVal*(tree: PackedTree; n: NodePos): string =
|
||||
assert n.kind == nkStrLit
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
#result = cookedStrLit(raw)
|
||||
|
||||
proc filenameVal*(tree: PackedTree; n: NodePos): string =
|
||||
case n.kind
|
||||
of nkStrLit:
|
||||
result = strVal(tree, n)
|
||||
of nkIdent:
|
||||
result = tree.sh.strings[n.litId]
|
||||
of nkSym:
|
||||
result = tree.sh.strings[tree.sh.syms[int n.symId].name]
|
||||
else:
|
||||
result = ""
|
||||
proc filenameVal*(tree: PackedTree; n: NodePos): string =
|
||||
case n.kind
|
||||
of nkStrLit:
|
||||
result = strVal(tree, n)
|
||||
of nkIdent:
|
||||
result = tree.sh.strings[n.litId]
|
||||
of nkSym:
|
||||
result = tree.sh.strings[tree.sh.syms[int n.symId].name]
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
|
||||
assert n.kind == nkIdent
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
|
||||
assert n.kind == nkIdent
|
||||
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
|
||||
|
||||
const
|
||||
externIntLit* = {nkCharLit,
|
||||
|
|
@ -380,7 +410,8 @@ const
|
|||
externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
|
||||
directIntLit* = nkInt32Lit
|
||||
|
||||
proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
|
||||
proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int;
|
||||
result: var string) =
|
||||
let pos = n.int
|
||||
if result.len > 0 and result[^1] notin {' ', '\n'}:
|
||||
result.add ' '
|
||||
|
|
@ -390,46 +421,47 @@ proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
|
|||
of nkNone, nkEmpty, nkNilLit, nkType: discard
|
||||
of nkIdent, nkStrLit..nkTripleStrLit:
|
||||
result.add " "
|
||||
result.add tree.sh.strings[LitId tree.nodes[pos].operand]
|
||||
result.add sh.strings[LitId tree.nodes[pos].operand]
|
||||
of nkSym:
|
||||
result.add " "
|
||||
result.add tree.sh.strings[tree.sh.syms[tree.nodes[pos].operand].name]
|
||||
result.add sh.strings[sh.syms[tree.nodes[pos].operand].name]
|
||||
of directIntLit:
|
||||
result.add " "
|
||||
result.addInt tree.nodes[pos].operand
|
||||
of externSIntLit:
|
||||
result.add " "
|
||||
result.addInt tree.sh.integers[LitId tree.nodes[pos].operand]
|
||||
result.addInt sh.integers[LitId tree.nodes[pos].operand]
|
||||
of externUIntLit:
|
||||
result.add " "
|
||||
result.add $cast[uint64](tree.sh.integers[LitId tree.nodes[pos].operand])
|
||||
result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand])
|
||||
else:
|
||||
result.add "(\n"
|
||||
for i in 1..(nesting+1)*2: result.add ' '
|
||||
for child in sonsReadonly(tree, n):
|
||||
toString(tree, child, nesting + 1, result)
|
||||
toString(tree, child, sh, nesting + 1, result)
|
||||
result.add "\n"
|
||||
for i in 1..nesting*2: result.add ' '
|
||||
result.add ")"
|
||||
#for i in 1..nesting*2: result.add ' '
|
||||
|
||||
|
||||
proc toString*(tree: PackedTree; n: NodePos): string =
|
||||
proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string =
|
||||
result = ""
|
||||
toString(tree, n, 0, result)
|
||||
toString(tree, n, sh, 0, result)
|
||||
|
||||
proc debug*(tree: PackedTree) =
|
||||
stdout.write toString(tree, NodePos 0)
|
||||
proc debug*(tree: PackedTree; sh: Shared) =
|
||||
stdout.write toString(tree, NodePos 0, sh)
|
||||
|
||||
proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
|
||||
if n.kind == nkIdent:
|
||||
result = n.litId
|
||||
elif n.kind == nkSym:
|
||||
result = tree.sh.syms[int n.symId].name
|
||||
else:
|
||||
result = LitId(0)
|
||||
when false:
|
||||
proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
|
||||
if n.kind == nkIdent:
|
||||
result = n.litId
|
||||
elif n.kind == nkSym:
|
||||
result = tree.sh.syms[int n.symId].name
|
||||
else:
|
||||
result = LitId(0)
|
||||
|
||||
template identId*(n: NodePos): LitId = identIdImpl(tree, n)
|
||||
template identId*(n: NodePos): LitId = identIdImpl(tree, n)
|
||||
|
||||
template copyInto*(dest, n, body) =
|
||||
let patchPos = prepare(dest, tree, n)
|
||||
|
|
@ -441,17 +473,20 @@ template copyIntoKind*(dest, kind, info, body) =
|
|||
body
|
||||
patch dest, patchPos
|
||||
|
||||
proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
|
||||
let litId = tree.sh.strings.getKeyId(pragma)
|
||||
if litId == LitId(0):
|
||||
return false
|
||||
assert n.kind == nkPragma
|
||||
for ch0 in sonsReadonly(tree, n):
|
||||
if ch0.kind == nkExprColonExpr:
|
||||
if ch0.firstSon.identId == litId:
|
||||
when false:
|
||||
proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
|
||||
let litId = tree.sh.strings.getKeyId(pragma)
|
||||
if litId == LitId(0):
|
||||
return false
|
||||
assert n.kind == nkPragma
|
||||
for ch0 in sonsReadonly(tree, n):
|
||||
if ch0.kind == nkExprColonExpr:
|
||||
if ch0.firstSon.identId == litId:
|
||||
return true
|
||||
elif ch0.identId == litId:
|
||||
return true
|
||||
elif ch0.identId == litId:
|
||||
return true
|
||||
|
||||
proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
|
||||
|
||||
when false:
|
||||
proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
|
||||
|
|
@ -459,3 +494,68 @@ when false:
|
|||
dest.add nkStrLit, msg, n.info
|
||||
copyTree(dest, tree, n)
|
||||
patch dest, patchPos
|
||||
|
||||
proc hash*(table: StringTableRef): Hash =
|
||||
## XXX: really should be introduced into strtabs...
|
||||
var h: Hash = 0
|
||||
for pair in pairs table:
|
||||
h = h !& hash(pair)
|
||||
result = !$h
|
||||
|
||||
proc hash*(config: ConfigRef): Hash =
|
||||
## XXX: vet and/or extend this
|
||||
var h: Hash = 0
|
||||
h = h !& hash(config.selectedGC)
|
||||
h = h !& hash(config.features)
|
||||
h = h !& hash(config.legacyFeatures)
|
||||
h = h !& hash(config.configVars)
|
||||
h = h !& hash(config.symbols)
|
||||
result = !$h
|
||||
|
||||
# XXX: lazy hashes for now
|
||||
type
|
||||
LazyHashes = PackedSym or PackedType or PackedLib or
|
||||
PackedLineInfo or PackedTree or PackedNode
|
||||
|
||||
proc hash*(sh: Shared): Hash
|
||||
proc hash*(s: LazyHashes): Hash
|
||||
proc hash*(s: seq[LazyHashes]): Hash
|
||||
|
||||
proc hash*(s: LazyHashes): Hash =
|
||||
var h: Hash = 0
|
||||
for k, v in fieldPairs(s):
|
||||
h = h !& hash((k, v))
|
||||
result = !$h
|
||||
|
||||
proc hash*(s: seq[LazyHashes]): Hash =
|
||||
## critically, we need to hash the indices alongside their values
|
||||
var h: Hash = 0
|
||||
for i, n in pairs s:
|
||||
h = h !& hash((i, n))
|
||||
result = !$h
|
||||
|
||||
proc hash*(sh: Shared): Hash =
|
||||
## might want to edit this...
|
||||
# XXX: these have too many references
|
||||
when false:
|
||||
var h: Hash = 0
|
||||
h = h !& hash(sh.syms)
|
||||
h = h !& hash(sh.types)
|
||||
h = h !& hash(sh.strings)
|
||||
h = h !& hash(sh.integers)
|
||||
h = h !& hash(sh.floats)
|
||||
h = h !& hash(sh.config)
|
||||
result = !$h
|
||||
|
||||
proc hash*(m: Module): Hash =
|
||||
var h: Hash = 0
|
||||
h = h !& hash(m.name)
|
||||
h = h !& hash(m.ast)
|
||||
result = !$h
|
||||
|
||||
template safeItemId*(x: typed; f: untyped): ItemId =
|
||||
## yield a valid ItemId value for the field of a nillable type
|
||||
if x.isNil:
|
||||
nilItemId
|
||||
else:
|
||||
x.`f`
|
||||
|
|
|
|||
143
compiler/ic/rodfiles.nim
Normal file
143
compiler/ic/rodfiles.nim
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
#
|
||||
#
|
||||
# The Nim Compiler
|
||||
# (c) Copyright 2020 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
from typetraits import supportsCopyMem
|
||||
|
||||
type
|
||||
RodSection* = enum
|
||||
versionSection
|
||||
configSection
|
||||
stringsSection
|
||||
checkSumsSection
|
||||
depsSection
|
||||
integersSection
|
||||
floatsSection
|
||||
topLevelSection
|
||||
bodiesSection
|
||||
symsSection
|
||||
typesSection
|
||||
|
||||
RodFileError* = enum
|
||||
ok, tooBig, ioFailure, wrongHeader, wrongSection, configMismatch,
|
||||
includeFileChanged
|
||||
|
||||
RodFile* = object
|
||||
f*: File
|
||||
currentSection*: RodSection # for error checking
|
||||
err*: RodFileError # little experiment to see if this works
|
||||
# better than exceptions.
|
||||
|
||||
const
|
||||
RodVersion = 1
|
||||
cookie = [byte(0), byte('R'), byte('O'), byte('D'),
|
||||
byte(0), byte(0), byte(0), byte(RodVersion)]
|
||||
|
||||
proc storePrim*(f: var RodFile; s: string) =
|
||||
if f.err != ok: return
|
||||
if s.len >= high(int32):
|
||||
f.err = tooBig
|
||||
return
|
||||
var lenPrefix = int32(s.len)
|
||||
if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
|
||||
f.err = ioFailure
|
||||
else:
|
||||
if s.len != 0:
|
||||
if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
|
||||
f.err = ioFailure
|
||||
|
||||
proc storePrim*[T](f: var RodFile; x: T) =
|
||||
if f.err != ok: return
|
||||
when supportsCopyMem(T):
|
||||
if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
|
||||
f.err = ioFailure
|
||||
elif T is tuple:
|
||||
for y in fields(x):
|
||||
storePrim(f, y)
|
||||
else:
|
||||
{.error: "unsupported type for 'storePrim'".}
|
||||
|
||||
proc storeSeq*[T](f: var RodFile; s: seq[T]) =
|
||||
if f.err != ok: return
|
||||
if s.len >= high(int32):
|
||||
f.err = tooBig
|
||||
return
|
||||
var lenPrefix = int32(s.len)
|
||||
if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
|
||||
f.err = ioFailure
|
||||
else:
|
||||
for i in 0..<s.len:
|
||||
storePrim(f, s[i])
|
||||
|
||||
proc loadPrim*(f: var RodFile; s: var string) =
|
||||
if f.err != ok: return
|
||||
var lenPrefix = int32(0)
|
||||
if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
|
||||
f.err = ioFailure
|
||||
else:
|
||||
s = newString(lenPrefix)
|
||||
if lenPrefix > 0:
|
||||
if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
|
||||
f.err = ioFailure
|
||||
|
||||
proc loadPrim*[T](f: var RodFile; x: var T) =
|
||||
if f.err != ok: return
|
||||
when supportsCopyMem(T):
|
||||
if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
|
||||
f.err = ioFailure
|
||||
elif T is tuple:
|
||||
for y in fields(x):
|
||||
loadPrim(f, y)
|
||||
else:
|
||||
{.error: "unsupported type for 'loadPrim'".}
|
||||
|
||||
proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
|
||||
if f.err != ok: return
|
||||
var lenPrefix = int32(0)
|
||||
if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
|
||||
f.err = ioFailure
|
||||
else:
|
||||
s = newSeq[T](lenPrefix)
|
||||
for i in 0..<lenPrefix:
|
||||
loadPrim(f, s[i])
|
||||
|
||||
proc storeHeader*(f: var RodFile) =
|
||||
if f.err != ok: return
|
||||
if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
|
||||
f.err = ioFailure
|
||||
|
||||
proc loadHeader*(f: var RodFile) =
|
||||
if f.err != ok: return
|
||||
var thisCookie: array[cookie.len, byte]
|
||||
if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
|
||||
f.err = ioFailure
|
||||
elif thisCookie != cookie:
|
||||
f.err = wrongHeader
|
||||
|
||||
proc storeSection*(f: var RodFile; s: RodSection) =
|
||||
if f.err != ok: return
|
||||
assert f.currentSection == pred s
|
||||
f.currentSection = s
|
||||
storePrim(f, s)
|
||||
|
||||
proc loadSection*(f: var RodFile; expected: RodSection) =
|
||||
if f.err != ok: return
|
||||
var s: RodSection
|
||||
loadPrim(f, s)
|
||||
if expected != s:
|
||||
f.err = wrongSection
|
||||
|
||||
proc create*(filename: string): RodFile =
|
||||
if not open(result.f, filename, fmWrite):
|
||||
result.err = ioFailure
|
||||
|
||||
proc close*(f: var RodFile) = close(f.f)
|
||||
|
||||
proc open*(filename: string): RodFile =
|
||||
if not open(result.f, filename, fmRead):
|
||||
result.err = ioFailure
|
||||
|
|
@ -7,84 +7,680 @@
|
|||
# distribution, for details about the copyright.
|
||||
#
|
||||
|
||||
import std / [hashes, tables]
|
||||
import packed_ast, bitabs
|
||||
import ".." / [ast, idents, lineinfos, options, pathutils, msgs]
|
||||
import std / [hashes, tables, intsets, sha1]
|
||||
import packed_ast, bitabs, rodfiles
|
||||
import ".." / [ast, idents, lineinfos, msgs, ropes, options,
|
||||
pathutils, condsyms]
|
||||
|
||||
from std / os import removeFile, isAbsolute
|
||||
|
||||
when not defined(release): import ".." / astalgo # debug()
|
||||
|
||||
type
|
||||
Context = object
|
||||
thisModule: int32
|
||||
lastFile: FileIndex # remember the last lookup entry.
|
||||
lastLit: LitId
|
||||
filenames: Table[FileIndex, LitId]
|
||||
PackedConfig* = object
|
||||
backend: TBackend
|
||||
selectedGC: TGCMode
|
||||
cCompiler: TSystemCC
|
||||
options: TOptions
|
||||
globalOptions: TGlobalOptions
|
||||
|
||||
proc toLitId(x: FileIndex; ir: var PackedTree; c: var Context): LitId =
|
||||
PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file
|
||||
definedSymbols: string
|
||||
includes: seq[(LitId, string)] # first entry is the module filename itself
|
||||
imports: seq[LitId] # the modules this module depends on
|
||||
topLevel*: PackedTree # top level statements
|
||||
bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
|
||||
hidden*: PackedTree # instantiated generics and other trees not directly in the source code.
|
||||
#producedGenerics*: Table[GenericKey, SymId]
|
||||
sh*: Shared
|
||||
cfg: PackedConfig
|
||||
|
||||
PackedEncoder* = object
|
||||
m: PackedModule
|
||||
thisModule*: int32
|
||||
lastFile*: FileIndex # remember the last lookup entry.
|
||||
lastLit*: LitId
|
||||
filenames*: Table[FileIndex, LitId]
|
||||
pendingTypes*: seq[PType]
|
||||
pendingSyms*: seq[PSym]
|
||||
typeMarker*: IntSet #Table[ItemId, TypeId] # ItemId.item -> TypeId
|
||||
symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId
|
||||
config*: ConfigRef
|
||||
|
||||
template primConfigFields(fn: untyped) {.dirty.} =
|
||||
fn backend
|
||||
fn selectedGC
|
||||
fn cCompiler
|
||||
fn options
|
||||
fn globalOptions
|
||||
|
||||
proc definedSymbolsAsString(config: ConfigRef): string =
|
||||
result = newStringOfCap(200)
|
||||
result.add "config"
|
||||
for d in definedSymbolNames(config.symbols):
|
||||
result.add ' '
|
||||
result.add d
|
||||
|
||||
proc rememberConfig(c: var PackedEncoder; config: ConfigRef) =
|
||||
c.m.definedSymbols = definedSymbolsAsString(config)
|
||||
|
||||
template rem(x) =
|
||||
c.m.cfg.x = config.x
|
||||
primConfigFields rem
|
||||
|
||||
proc configIdentical(m: PackedModule; config: ConfigRef): bool =
|
||||
result = m.definedSymbols == definedSymbolsAsString(config)
|
||||
template eq(x) =
|
||||
result = result and m.cfg.x == config.x
|
||||
primConfigFields eq
|
||||
|
||||
proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string =
|
||||
result = msgs.getHash(conf, fileIdx)
|
||||
if result.len == 0:
|
||||
let fullpath = msgs.toFullPath(conf, fileIdx)
|
||||
result = $secureHashFile(fullpath)
|
||||
msgs.setHash(conf, fileIdx, result)
|
||||
|
||||
proc toLitId(x: FileIndex; c: var PackedEncoder): LitId =
|
||||
## store a file index as a literal
|
||||
if x == c.lastFile:
|
||||
result = c.lastLit
|
||||
else:
|
||||
result = c.filenames.getOrDefault(x)
|
||||
if result == LitId(0):
|
||||
let p = msgs.toFullPath(ir.sh.config, x)
|
||||
result = getOrIncl(ir.sh.strings, p)
|
||||
let p = msgs.toFullPath(c.config, x)
|
||||
result = getOrIncl(c.m.sh.strings, p)
|
||||
c.filenames[x] = result
|
||||
c.lastFile = x
|
||||
c.lastLit = result
|
||||
assert result != LitId(0)
|
||||
|
||||
proc toPackedInfo(x: TLineInfo; ir: var PackedTree; c: var Context): PackedLineInfo =
|
||||
PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, ir, c))
|
||||
proc toFileIndex(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
|
||||
result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
|
||||
|
||||
proc toPackedType(t: PType; ir: var PackedTree; c: var Context): TypeId =
|
||||
result = TypeId(0)
|
||||
proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
|
||||
for it in mitems(m.includes):
|
||||
if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]:
|
||||
return false
|
||||
result = true
|
||||
|
||||
proc toPackedSym(s: PSym; ir: var PackedTree; c: var Context): SymId =
|
||||
result = SymId(0)
|
||||
proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
|
||||
## setup a context for serializing to packed ast
|
||||
c.m.sh = Shared()
|
||||
c.thisModule = m.itemId.module
|
||||
c.config = config
|
||||
c.m.bodies = newTreeFrom(c.m.topLevel)
|
||||
c.m.hidden = newTreeFrom(c.m.topLevel)
|
||||
|
||||
proc toPackedSymNode(n: PNode; ir: var PackedTree; c: var Context) =
|
||||
let thisNimFile = FileIndex c.thisModule
|
||||
var h = msgs.getHash(config, thisNimFile)
|
||||
if h.len == 0:
|
||||
let fullpath = msgs.toFullPath(config, thisNimFile)
|
||||
if isAbsolute(fullpath):
|
||||
# For NimScript compiler API support the main Nim file might be from a stream.
|
||||
h = $secureHashFile(fullpath)
|
||||
msgs.setHash(config, thisNimFile, h)
|
||||
c.m.includes.add((toLitId(thisNimFile, c), h)) # the module itself
|
||||
|
||||
proc addIncludeFileDep*(c: var PackedEncoder; f: FileIndex) =
|
||||
c.m.includes.add((toLitId(f, c), hashFileCached(c.config, f)))
|
||||
|
||||
proc addImportFileDep*(c: var PackedEncoder; f: FileIndex) =
|
||||
c.m.imports.add toLitId(f, c)
|
||||
|
||||
proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder)
|
||||
proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId
|
||||
proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId
|
||||
|
||||
proc flush(c: var PackedEncoder) =
|
||||
## serialize any pending types or symbols from the context
|
||||
while true:
|
||||
if c.pendingTypes.len > 0:
|
||||
discard toPackedType(c.pendingTypes.pop, c)
|
||||
elif c.pendingSyms.len > 0:
|
||||
discard toPackedSym(c.pendingSyms.pop, c)
|
||||
else:
|
||||
break
|
||||
|
||||
proc toLitId(x: string; c: var PackedEncoder): LitId =
|
||||
## store a string as a literal
|
||||
result = getOrIncl(c.m.sh.strings, x)
|
||||
|
||||
proc toLitId(x: BiggestInt; c: var PackedEncoder): LitId =
|
||||
## store an integer as a literal
|
||||
result = getOrIncl(c.m.sh.integers, x)
|
||||
|
||||
proc toPackedInfo(x: TLineInfo; c: var PackedEncoder): PackedLineInfo =
|
||||
PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c))
|
||||
|
||||
proc safeItemId(s: PSym; c: var PackedEncoder): PackedItemId {.inline.} =
|
||||
## given a symbol, produce an ItemId with the correct properties
|
||||
## for local or remote symbols, packing the symbol as necessary
|
||||
if s == nil:
|
||||
result = nilItemId
|
||||
elif s.itemId.module == c.thisModule:
|
||||
result = PackedItemId(module: LitId(0), item: s.itemId.item)
|
||||
else:
|
||||
result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c),
|
||||
item: s.itemId.item)
|
||||
|
||||
proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
|
||||
## add a remote symbol reference to the tree
|
||||
let info = n.info.toPackedInfo(c)
|
||||
ir.nodes.add PackedNode(kind: nkModuleRef, operand: 2.int32, # 2 kids...
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
|
||||
operand: toLitId(n.sym.itemId.module.FileIndex, c).int32)
|
||||
ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
|
||||
operand: n.sym.itemId.item)
|
||||
|
||||
proc addMissing(c: var PackedEncoder; p: PSym) =
|
||||
## consider queuing a symbol for later addition to the packed tree
|
||||
if p != nil and p.itemId.module == c.thisModule:
|
||||
if p.itemId.item notin c.symMarker:
|
||||
c.pendingSyms.add p
|
||||
|
||||
proc addMissing(c: var PackedEncoder; p: PType) =
|
||||
## consider queuing a type for later addition to the packed tree
|
||||
if p != nil and p.uniqueId.module == c.thisModule:
|
||||
if p.uniqueId.item notin c.typeMarker:
|
||||
c.pendingTypes.add p
|
||||
|
||||
template storeNode(dest, src, field) =
|
||||
var nodeId: NodeId
|
||||
if src.field != nil:
|
||||
nodeId = getNodeId(c.m.bodies)
|
||||
toPackedNode(src.field, c.m.bodies, c)
|
||||
else:
|
||||
nodeId = emptyNodeId
|
||||
dest.field = nodeId
|
||||
|
||||
proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId =
|
||||
## serialize a ptype
|
||||
if t.isNil: return nilTypeId
|
||||
|
||||
if t.uniqueId.module != c.thisModule:
|
||||
# XXX Assert here that it already was serialized in the foreign module!
|
||||
# it is a foreign type:
|
||||
return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c), item: t.uniqueId.item)
|
||||
|
||||
if not c.typeMarker.containsOrIncl(t.uniqueId.item):
|
||||
if t.uniqueId.item >= c.m.sh.types.len:
|
||||
setLen c.m.sh.types, t.uniqueId.item+1
|
||||
|
||||
var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
|
||||
size: t.size, align: t.align, nonUniqueId: t.itemId.item,
|
||||
paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
|
||||
storeNode(p, t, n)
|
||||
|
||||
for op, s in pairs t.attachedOps:
|
||||
c.addMissing s
|
||||
p.attachedOps[op] = s.safeItemId(c)
|
||||
|
||||
p.typeInst = t.typeInst.toPackedType(c)
|
||||
for kid in items t.sons:
|
||||
p.types.add kid.toPackedType(c)
|
||||
for i, s in items t.methods:
|
||||
c.addMissing s
|
||||
p.methods.add (i, s.safeItemId(c))
|
||||
c.addMissing t.sym
|
||||
p.sym = t.sym.safeItemId(c)
|
||||
c.addMissing t.owner
|
||||
p.owner = t.owner.safeItemId(c)
|
||||
|
||||
# fill the reserved slot, nothing else:
|
||||
c.m.sh.types[t.uniqueId.item] = p
|
||||
|
||||
result = PackedItemId(module: LitId(0), item: t.uniqueId.item)
|
||||
|
||||
proc toPackedLib(l: PLib; c: var PackedEncoder): PackedLib =
|
||||
## the plib hangs off the psym via the .annex field
|
||||
if l.isNil: return
|
||||
result.kind = l.kind
|
||||
result.generated = l.generated
|
||||
result.isOverriden = l.isOverriden
|
||||
result.name = toLitId($l.name, c)
|
||||
storeNode(result, l, path)
|
||||
|
||||
proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId =
|
||||
## serialize a psym
|
||||
if s.isNil: return nilItemId
|
||||
|
||||
if s.itemId.module != c.thisModule:
|
||||
# XXX Assert here that it already was serialized in the foreign module!
|
||||
# it is a foreign symbol:
|
||||
return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c), item: s.itemId.item)
|
||||
|
||||
if not c.symMarker.containsOrIncl(s.itemId.item):
|
||||
if s.itemId.item >= c.m.sh.syms.len:
|
||||
setLen c.m.sh.syms, s.itemId.item+1
|
||||
|
||||
var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c), magic: s.magic,
|
||||
position: s.position, offset: s.offset, options: s.options,
|
||||
name: s.name.s.toLitId(c))
|
||||
|
||||
storeNode(p, s, ast)
|
||||
storeNode(p, s, constraint)
|
||||
|
||||
if s.kind in {skLet, skVar, skField, skForVar}:
|
||||
c.addMissing s.guard
|
||||
p.guard = s.guard.safeItemId(c)
|
||||
p.bitsize = s.bitsize
|
||||
p.alignment = s.alignment
|
||||
|
||||
p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, c)
|
||||
c.addMissing s.typ
|
||||
p.typ = s.typ.toPackedType(c)
|
||||
c.addMissing s.owner
|
||||
p.owner = s.owner.safeItemId(c)
|
||||
p.annex = toPackedLib(s.annex, c)
|
||||
when hasFFI:
|
||||
p.cname = toLitId(s.cname, c)
|
||||
|
||||
# fill the reserved slot, nothing else:
|
||||
c.m.sh.syms[s.itemId.item] = p
|
||||
|
||||
result = PackedItemId(module: LitId(0), item: s.itemId.item)
|
||||
|
||||
proc toSymNode(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
|
||||
## store a local or remote psym reference in the tree
|
||||
assert n.kind == nkSym
|
||||
let t = toPackedType(n.typ, ir, c)
|
||||
|
||||
if n.sym.itemId.module == c.thisModule:
|
||||
template s: PSym = n.sym
|
||||
let id = s.toPackedSym(c).item
|
||||
if s.itemId.module == c.thisModule:
|
||||
# it is a symbol that belongs to the module we're currently
|
||||
# packing:
|
||||
let sid = toPackedSym(n.sym, ir, c)
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(sid),
|
||||
typeId: t, info: toPackedInfo(n.info, ir, c))
|
||||
ir.addSym(id, toPackedInfo(n.info, c))
|
||||
else:
|
||||
# store it as an external module reference:
|
||||
# nkModuleRef
|
||||
discard
|
||||
|
||||
|
||||
proc toPackedNode*(n: PNode; ir: var PackedTree; c: var Context) =
|
||||
template toP(x: TLineInfo): PackedLineInfo = toPackedInfo(x, ir, c)
|
||||
addModuleRef(n, ir, c)
|
||||
|
||||
proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
|
||||
## serialize a node into the tree
|
||||
if n.isNil: return
|
||||
let info = toPackedInfo(n.info, c)
|
||||
case n.kind
|
||||
of nkNone, nkEmpty, nkNilLit:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: 0,
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
of nkNone, nkEmpty, nkNilLit, nkType:
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0,
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
of nkIdent:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.ident.s),
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
|
||||
operand: int32 getOrIncl(c.m.sh.strings, n.ident.s),
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
of nkSym:
|
||||
toPackedSymNode(n, ir, c)
|
||||
toSymNode(n, ir, c)
|
||||
of directIntLit:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(n.intVal),
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
|
||||
operand: int32(n.intVal),
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
of externIntLit:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.integers, n.intVal),
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
|
||||
operand: int32 getOrIncl(c.m.sh.integers, n.intVal),
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
of nkStrLit..nkTripleStrLit:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.strVal),
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
|
||||
operand: int32 getOrIncl(c.m.sh.strings, n.strVal),
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
of nkFloatLit..nkFloat128Lit:
|
||||
ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.floats, n.floatVal),
|
||||
typeId: toPackedType(n.typ, ir, c), info: toP n.info)
|
||||
ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
|
||||
operand: int32 getOrIncl(c.m.sh.floats, n.floatVal),
|
||||
typeId: toPackedType(n.typ, c), info: info)
|
||||
else:
|
||||
let patchPos = ir.prepare(n.kind, n.flags, toPackedType(n.typ, ir, c), toP n.info)
|
||||
let patchPos = ir.prepare(n.kind, n.flags,
|
||||
toPackedType(n.typ, c), info)
|
||||
for i in 0..<n.len:
|
||||
toPackedNode(n[i], ir, c)
|
||||
ir.patch patchPos
|
||||
|
||||
proc moduleToIr*(n: PNode; ir: var PackedTree; module: PSym) =
|
||||
var c = Context(thisModule: module.itemId.module)
|
||||
toPackedNode(n, ir, c)
|
||||
when false:
|
||||
ir.flush c # flush any pending types and symbols
|
||||
|
||||
proc toPackedNodeIgnoreProcDefs*(n: PNode, encoder: var PackedEncoder) =
|
||||
case n.kind
|
||||
of routineDefs:
|
||||
# we serialize n[namePos].sym instead
|
||||
if n[namePos].kind == nkSym:
|
||||
discard toPackedSym(n[namePos].sym, encoder)
|
||||
else:
|
||||
toPackedNode(n, encoder.m.topLevel, encoder)
|
||||
else:
|
||||
toPackedNode(n, encoder.m.topLevel, encoder)
|
||||
|
||||
proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder) =
|
||||
toPackedNodeIgnoreProcDefs(n, encoder)
|
||||
flush encoder
|
||||
|
||||
proc storePrim*(f: var RodFile; x: PackedType) =
|
||||
for y in fields(x):
|
||||
when y is seq:
|
||||
storeSeq(f, y)
|
||||
else:
|
||||
storePrim(f, y)
|
||||
|
||||
proc loadPrim*(f: var RodFile; x: var PackedType) =
|
||||
for y in fields(x):
|
||||
when y is seq:
|
||||
loadSeq(f, y)
|
||||
else:
|
||||
loadPrim(f, y)
|
||||
|
||||
proc loadError(err: RodFileError; filename: AbsoluteFile) =
|
||||
echo "Error: ", $err, "\nloading file: ", filename.string
|
||||
|
||||
proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef): RodFileError =
|
||||
m.sh = Shared()
|
||||
var f = rodfiles.open(filename.string)
|
||||
f.loadHeader()
|
||||
f.loadSection configSection
|
||||
|
||||
f.loadPrim m.definedSymbols
|
||||
f.loadPrim m.cfg
|
||||
|
||||
if not configIdentical(m, config):
|
||||
f.err = configMismatch
|
||||
|
||||
f.loadSection stringsSection
|
||||
f.load m.sh.strings
|
||||
|
||||
f.loadSection checkSumsSection
|
||||
f.loadSeq m.includes
|
||||
if not includesIdentical(m, config):
|
||||
f.err = includeFileChanged
|
||||
|
||||
f.loadSection depsSection
|
||||
f.loadSeq m.imports
|
||||
|
||||
f.loadSection integersSection
|
||||
f.load m.sh.integers
|
||||
f.loadSection floatsSection
|
||||
f.load m.sh.floats
|
||||
|
||||
f.loadSection topLevelSection
|
||||
f.loadSeq m.topLevel.nodes
|
||||
|
||||
f.loadSection bodiesSection
|
||||
f.loadSeq m.bodies.nodes
|
||||
|
||||
f.loadSection symsSection
|
||||
f.loadSeq m.sh.syms
|
||||
|
||||
f.loadSection typesSection
|
||||
f.loadSeq m.sh.types
|
||||
|
||||
close(f)
|
||||
result = f.err
|
||||
|
||||
type
|
||||
ModuleStatus* = enum
|
||||
undefined,
|
||||
loading,
|
||||
loaded,
|
||||
outdated
|
||||
|
||||
LoadedModule* = object
|
||||
status: ModuleStatus
|
||||
symsInit, typesInit: bool
|
||||
fromDisk: PackedModule
|
||||
syms: seq[PSym] # indexed by itemId
|
||||
types: seq[PType]
|
||||
|
||||
PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
|
||||
|
||||
proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef;
|
||||
fileIdx: FileIndex): bool =
|
||||
let m = int(fileIdx)
|
||||
if m >= g.len:
|
||||
g.setLen(m+1)
|
||||
|
||||
case g[m].status
|
||||
of undefined:
|
||||
g[m].status = loading
|
||||
let fullpath = msgs.toFullPath(conf, fileIdx)
|
||||
let rod = toRodFile(conf, AbsoluteFile fullpath)
|
||||
let err = loadRodFile(rod, g[m].fromDisk, conf)
|
||||
if err == ok:
|
||||
result = false
|
||||
# check its dependencies:
|
||||
for dep in g[m].fromDisk.imports:
|
||||
let fid = toFileIndex(dep, g[m].fromDisk, conf)
|
||||
# Warning: we need to traverse the full graph, so
|
||||
# do **not use break here**!
|
||||
if needsRecompile(g, conf, fid):
|
||||
result = true
|
||||
|
||||
g[m].status = if result: outdated else: loaded
|
||||
else:
|
||||
loadError(err, rod)
|
||||
g[m].status = outdated
|
||||
result = true
|
||||
of loading, loaded:
|
||||
result = false
|
||||
of outdated:
|
||||
result = true
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
proc storeError(err: RodFileError; filename: AbsoluteFile) =
|
||||
echo "Error: ", $err, "; couldn't write to ", filename.string
|
||||
removeFile(filename.string)
|
||||
|
||||
proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
|
||||
rememberConfig(encoder, encoder.config)
|
||||
|
||||
var f = rodfiles.create(filename.string)
|
||||
f.storeHeader()
|
||||
f.storeSection configSection
|
||||
f.storePrim encoder.m.definedSymbols
|
||||
f.storePrim encoder.m.cfg
|
||||
|
||||
f.storeSection stringsSection
|
||||
f.store encoder.m.sh.strings
|
||||
|
||||
f.storeSection checkSumsSection
|
||||
f.storeSeq encoder.m.includes
|
||||
|
||||
f.storeSection depsSection
|
||||
f.storeSeq encoder.m.imports
|
||||
|
||||
f.storeSection integersSection
|
||||
f.store encoder.m.sh.integers
|
||||
|
||||
f.storeSection floatsSection
|
||||
f.store encoder.m.sh.floats
|
||||
|
||||
f.storeSection topLevelSection
|
||||
f.storeSeq encoder.m.topLevel.nodes
|
||||
|
||||
f.storeSection bodiesSection
|
||||
f.storeSeq encoder.m.bodies.nodes
|
||||
|
||||
f.storeSection symsSection
|
||||
f.storeSeq encoder.m.sh.syms
|
||||
|
||||
f.storeSection typesSection
|
||||
f.storeSeq encoder.m.sh.types
|
||||
close(f)
|
||||
if f.err != ok:
|
||||
loadError(f.err, filename)
|
||||
|
||||
when true:
|
||||
# basic loader testing:
|
||||
var m2: PackedModule
|
||||
discard loadRodFile(filename, m2, encoder.config)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
type
|
||||
PackedDecoder* = object
|
||||
thisModule*: int32
|
||||
lastLit*: LitId
|
||||
lastFile*: FileIndex # remember the last lookup entry.
|
||||
config*: ConfigRef
|
||||
ident: IdentCache
|
||||
|
||||
proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType
|
||||
proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym
|
||||
|
||||
proc toFileIndexCached(c: var PackedDecoder; g: var PackedModuleGraph; f: LitId): FileIndex =
|
||||
if c.lastLit == f:
|
||||
result = c.lastFile
|
||||
else:
|
||||
result = toFileIndex(f, g[c.thisModule].fromDisk, c.config)
|
||||
c.lastLit = f
|
||||
c.lastFile = result
|
||||
|
||||
proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
x: PackedLineInfo): TLineInfo =
|
||||
assert g[c.thisModule].status == loaded
|
||||
result = TLineInfo(line: x.line, col: x.col,
|
||||
fileIndex: toFileIndexCached(c, g, x.file))
|
||||
|
||||
proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
tree: PackedTree; n: NodePos): PNode =
|
||||
let k = n.kind
|
||||
result = newNodeIT(k, translateLineInfo(c, g, n.info),
|
||||
loadType(c, g, n.typ))
|
||||
result.flags = n.flags
|
||||
|
||||
case k
|
||||
of nkEmpty, nkNilLit, nkType:
|
||||
discard
|
||||
of nkIdent:
|
||||
result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId])
|
||||
of nkSym:
|
||||
result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
|
||||
of directIntLit:
|
||||
result.intVal = tree.nodes[n.int].operand
|
||||
of externIntLit:
|
||||
result.intVal = g[c.thisModule].fromDisk.sh.integers[n.litId]
|
||||
of nkStrLit..nkTripleStrLit:
|
||||
result.strVal = g[c.thisModule].fromDisk.sh.strings[n.litId]
|
||||
of nkFloatLit..nkFloat128Lit:
|
||||
result.floatVal = g[c.thisModule].fromDisk.sh.floats[n.litId]
|
||||
of nkModuleRef:
|
||||
let (n1, n2) = sons2(tree, n)
|
||||
assert n1.kind == nkInt32Lit
|
||||
assert n2.kind == nkInt32Lit
|
||||
transitionNoneToSym(result)
|
||||
result.sym = loadSym(c, g, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
|
||||
else:
|
||||
for n0 in sonsReadonly(tree, n):
|
||||
result.add loadNodes(c, g, tree, n0)
|
||||
|
||||
proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
s: PackedItemId): int32 {.inline.} =
|
||||
result = if s.module == LitId(0): c.thisModule
|
||||
else: toFileIndexCached(c, g, s.module).int32
|
||||
|
||||
proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
s: PackedSym; si, item: int32): PSym =
|
||||
result = PSym(itemId: ItemId(module: si, item: item),
|
||||
kind: s.kind, magic: s.magic, flags: s.flags,
|
||||
info: translateLineInfo(c, g, s.info),
|
||||
options: s.options,
|
||||
position: s.position,
|
||||
name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name])
|
||||
)
|
||||
|
||||
template loadAstBody(p, field) =
|
||||
if p.field != emptyNodeId:
|
||||
result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field)
|
||||
|
||||
proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
si, item: int32; l: PackedLib): PLib =
|
||||
# XXX: hack; assume a zero LitId means the PackedLib is all zero (empty)
|
||||
if l.name.int == 0:
|
||||
result = nil
|
||||
else:
|
||||
result = PLib(generated: l.generated, isOverriden: l.isOverriden,
|
||||
kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name])
|
||||
loadAstBody(l, path)
|
||||
|
||||
proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
s: PackedSym; si, item: int32; result: PSym) =
|
||||
result.typ = loadType(c, g, s.typ)
|
||||
loadAstBody(s, constraint)
|
||||
loadAstBody(s, ast)
|
||||
result.annex = loadLib(c, g, si, item, s.annex)
|
||||
when hasFFI:
|
||||
result.cname = g[si].fromDisk.sh.strings[s.cname]
|
||||
|
||||
if s.kind in {skLet, skVar, skField, skForVar}:
|
||||
result.guard = loadSym(c, g, s.guard)
|
||||
result.bitsize = s.bitsize
|
||||
result.alignment = s.alignment
|
||||
result.owner = loadSym(c, g, s.owner)
|
||||
let externalName = g[si].fromDisk.sh.strings[s.externalName]
|
||||
if externalName != "":
|
||||
result.loc.r = rope externalName
|
||||
|
||||
proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym =
|
||||
if s == nilTypeId:
|
||||
result = nil
|
||||
else:
|
||||
let si = moduleIndex(c, g, s)
|
||||
assert g[si].status == loaded
|
||||
if not g[si].symsInit:
|
||||
g[si].symsInit = true
|
||||
setLen g[si].syms, g[si].fromDisk.sh.syms.len
|
||||
|
||||
if g[si].syms[s.item] == nil:
|
||||
let packed = addr(g[si].fromDisk.sh.syms[s.item])
|
||||
result = symHeaderFromPacked(c, g, packed[], si, s.item)
|
||||
# store it here early on, so that recursions work properly:
|
||||
g[si].syms[s.item] = result
|
||||
symBodyFromPacked(c, g, packed[], si, s.item, result)
|
||||
else:
|
||||
result = g[si].syms[s.item]
|
||||
|
||||
proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
t: PackedType; si, item: int32): PType =
|
||||
result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind,
|
||||
flags: t.flags, size: t.size, align: t.align,
|
||||
paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel,
|
||||
uniqueId: ItemId(module: si, item: item))
|
||||
|
||||
proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
|
||||
t: PackedType; si, item: int32; result: PType) =
|
||||
result.sym = loadSym(c, g, t.sym)
|
||||
result.owner = loadSym(c, g, t.owner)
|
||||
for op, item in pairs t.attachedOps:
|
||||
result.attachedOps[op] = loadSym(c, g, item)
|
||||
result.typeInst = loadType(c, g, t.typeInst)
|
||||
for son in items t.types:
|
||||
result.sons.add loadType(c, g, son)
|
||||
loadAstBody(t, n)
|
||||
for gen, id in items t.methods:
|
||||
result.methods.add((gen, loadSym(c, g, id)))
|
||||
|
||||
proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType =
|
||||
if t == nilTypeId:
|
||||
result = nil
|
||||
else:
|
||||
let si = moduleIndex(c, g, t)
|
||||
assert g[si].status == loaded
|
||||
if not g[si].typesInit:
|
||||
g[si].typesInit = true
|
||||
setLen g[si].types, g[si].fromDisk.sh.types.len
|
||||
|
||||
if g[si].types[t.item] == nil:
|
||||
let packed = addr(g[si].fromDisk.sh.types[t.item])
|
||||
result = typeHeaderFromPacked(c, g, packed[], si, t.item)
|
||||
# store it here early on, so that recursions work properly:
|
||||
g[si].types[t.item] = result
|
||||
typeBodyFromPacked(c, g, packed[], si, t.item, result)
|
||||
else:
|
||||
result = g[si].types[t.item]
|
||||
|
||||
|
||||
when false:
|
||||
proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey =
|
||||
result.module = s.owner.itemId.module
|
||||
result.name = s.name.s
|
||||
result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest
|
||||
|
||||
proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) =
|
||||
## add a generic to the module
|
||||
if key notin m.generics:
|
||||
m.generics[key] = toPackedSym(s, m.ast, c)
|
||||
toPackedNode(s.ast, m.ast, c)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue