big steps torwards an efficient, simple IC implementation (#16543)

* reworked ID handling
* the packed AST now has its own ID mechanism
* basic serialization code works
* extract rodfiles to its own module
* rodfiles: store and compare configs
* rodfiles: store dependencies
* store config at the end
* precise dependency tracking
* dependency tracking for rodfiles
* completed loading of PSym, PType, etc
* removed dead code
* bugfix: do not realloc seqs when taking addr into an element
* make IC opt-in for now
* makes tcompilerapi green again
* final cleanups

Co-authored-by: Andy Davidoff <github@andy.disruptek.com>
This commit is contained in:
Andreas Rumpf 2021-01-02 07:30:39 +01:00 • committed by GitHub
commit 73a8b950cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 1429 additions and 1717 deletions

View file

@ -12,9 +12,9 @@
## use this representation directly in all the transformations,
## it is superior.
import std / [hashes, tables]
import std / [hashes, tables, strtabs, md5]
import bitabs
import ".." / [ast, lineinfos, options, pathutils]
import ".." / [ast, options]
const
localNamePos* = 0
@ -39,16 +39,29 @@ const
routineBodyPos* = 7
const
nkModuleRef = nkNone # pair of (ModuleId, SymId)
nkModuleRef* = nkNone # pair of (ModuleId, SymId)
type
SymId* = distinct int32
TypeId* = distinct int32
ModuleId* = distinct int32
NodePos* = distinct int
NodeId* = distinct int32
PackedItemId* = object
module*: LitId # 0 if it's this module
item*: int32 # same as the in-memory representation
TypeId* = PackedItemId
const
nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32)
nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
const
emptyNodeId* = NodeId(-1)
type
PackedLineInfo* = object
line*: uint16
col*: int16
@ -64,13 +77,13 @@ type
PackedSym* = object
kind*: TSymKind
name*: LitId
typeId*: TypeId
typ*: TypeId
flags*: TSymFlags
magic*: TMagic
info*: PackedLineInfo
ast*: NodePos
owner*: ItemId
guard*: ItemId
ast*: NodeId
owner*: PackedItemId
guard*: PackedItemId
bitsize*: int
alignment*: int # for alignment
options*: TOptions
@ -84,25 +97,25 @@ type
PackedType* = object
kind*: TTypeKind
nodekind*: TNodeKind
callConv*: TCallingConvention
#nodekind*: TNodeKind
flags*: TTypeFlags
types*: int32
nodes*: int32
methods*: int32
nodeflags*: TNodeFlags
info*: PackedLineInfo
sym*: ItemId
owner*: ItemId
attachedOps*: array[TTypeAttachedOp, ItemId]
types*: seq[TypeId]
n*: NodeId
methods*: seq[(int, PackedItemId)]
#nodeflags*: TNodeFlags
sym*: PackedItemId
owner*: PackedItemId
attachedOps*: array[TTypeAttachedOp, PackedItemId]
size*: BiggestInt
align*: int16
paddingAtEnd*: int16
lockLevel*: TLockLevel # lock level as required for deadlock checking
# not serialized: loc*: TLoc because it is backend-specific
typeInst*: TypeId
nonUniqueId*: ItemId
nonUniqueId*: int32
Node* = object # 20 bytes
PackedNode* = object # 20 bytes
kind*: TNodeKind
flags*: TNodeFlags
operand*: int32 # for kind in {nkSym, nkSymDef}: SymId
@ -115,72 +128,73 @@ type
ModulePhase* = enum
preLookup, lookedUpTopLevelStmts
Module* = object
GenericKey* = object
module*: int32
name*: string
file*: AbsoluteFile
ast*: PackedTree
phase*: ModulePhase
iface*: Table[string, seq[SymId]] # 'seq' because of overloading
types*: seq[MD5Digest] # is this a joke?
Program* = ref object
modules*: seq[Module]
PackedTree* = object ## usually represents a full Nim module
nodes*: seq[PackedNode]
#sh*: Shared
Shared* = ref object # shared between different versions of 'Module'.
# (though there is always exactly one valid
# version of a module)
syms*: seq[PackedSym]
types*: seq[seq[Node]]
types*: seq[PackedType]
strings*: BiTable[string] # we could share these between modules.
integers*: BiTable[BiggestInt]
floats*: BiTable[BiggestFloat]
config*: ConfigRef
#thisModule*: ModuleId
#program*: Program
#config*: ConfigRef
PackedTree* = object ## usually represents a full Nim module
nodes*: seq[Node]
toPosition*: Table[SymId, NodePos]
sh*: Shared
proc hash*(key: GenericKey): Hash =
var h: Hash = 0
h = h !& hash(key.module)
h = h !& hash(key.name)
h = h !& hash(key.types)
result = !$h
proc `==`*(a, b: SymId): bool {.borrow.}
proc hash*(a: SymId): Hash {.borrow.}
proc `==`*(a, b: NodePos): bool {.borrow.}
proc `==`*(a, b: TypeId): bool {.borrow.}
proc `==`*(a, b: ModuleId): bool {.borrow.}
proc declareSym*(tree: var PackedTree; kind: TSymKind;
name: LitId; info: PackedLineInfo): SymId =
result = SymId(tree.sh.syms.len)
tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
#proc `==`*(a, b: TypeId): bool {.borrow.}
proc `==`*(a, b: NodeId): bool {.borrow.}
proc newTreeFrom*(old: PackedTree): PackedTree =
result.nodes = @[]
result.sh = old.sh
when false: result.sh = old.sh
proc litIdFromName*(tree: PackedTree; name: string): LitId =
result = tree.sh.strings.getOrIncl(name)
when false:
proc declareSym*(tree: var PackedTree; kind: TSymKind;
name: LitId; info: PackedLineInfo): SymId =
result = SymId(tree.sh.syms.len)
tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
tree.nodes.add Node(kind: kind, operand: int32 getOrIncl(tree.sh.strings, token), info: info)
proc litIdFromName*(tree: PackedTree; name: string): LitId =
result = tree.sh.strings.getOrIncl(name)
proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
tree.nodes.add Node(kind: kind, operand: 0, info: info)
proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
tree.nodes.add PackedNode(kind: kind, info: info,
operand: int32 getOrIncl(tree.sh.strings, token))
proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
proc throwAwayLastNode*(tree: var PackedTree) =
tree.nodes.setLen(tree.nodes.len-1)
proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
tree.nodes.add Node(kind: nkIdent, operand: int32(s), info: info)
tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
proc addSym*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
tree.nodes.add Node(kind: nkInt32Lit, operand: int32(s), info: info)
tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
@ -194,11 +208,12 @@ proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
for i in 0..<L:
dest.nodes[d+i] = tree.nodes[pos+i]
proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
result = SymId(dest.sh.syms.len)
assert int(s) < tree.sh.syms.len
let oldSym = tree.sh.syms[s.int]
dest.sh.syms.add oldSym
when false:
proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
result = SymId(dest.sh.syms.len)
assert int(s) < tree.sh.syms.len
let oldSym = tree.sh.syms[s.int]
dest.sh.syms.add oldSym
type
PatchPos = distinct int
@ -206,11 +221,12 @@ type
when false:
proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
result = PatchPos tree.nodes.len
tree.nodes.add Node(kind: kind, operand: 0, info: info)
tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: TypeId; info: PackedLineInfo): PatchPos =
result = PatchPos tree.nodes.len
tree.nodes.add Node(kind: kind, flags: flags, operand: 0, typeId: typeId, info: info)
tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
typeId: typeId)
proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
result = PatchPos dest.nodes.len
@ -224,7 +240,8 @@ proc patch*(tree: var PackedTree; pos: PatchPos) =
proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
proc `[]`*(tree: PackedTree; i: int): lent Node {.inline.} = tree.nodes[i]
proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
tree.nodes[i]
proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
if tree.nodes[pos].kind > nkNilLit:
@ -247,7 +264,8 @@ iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
for x in sonsReadonly(tree, n): yield x
patch dest, patchPos
iterator isons*(dest: var PackedTree; tree: PackedTree; n: NodePos): (int, NodePos) =
iterator isons*(dest: var PackedTree; tree: PackedTree;
n: NodePos): (int, NodePos) =
var i = 0
for ch0 in sons(dest, tree, n):
yield (i, ch0)
@ -301,10 +319,19 @@ proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
if count >= x: return true
return false
proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind
proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = LitId tree.nodes[n.int].operand
proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info
proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
NodePos(n.int+1)
proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
tree.nodes[n.int].kind
proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
LitId tree.nodes[n.int].operand
proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
tree.nodes[n.int].info
template typ*(n: NodePos): PackedItemId =
tree.nodes[n.int].typeId
template flags*(n: NodePos): TNodeFlags =
tree.nodes[n.int].flags
proc span(tree: PackedTree; pos: int): int {.inline.} =
if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
@ -330,7 +357,9 @@ proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
inc count
assert false, "node has no i-th child"
proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = tree.sh.strings[lit]
when false:
proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} =
tree.sh.strings[lit]
template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
@ -340,29 +369,30 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
proc strLit*(tree: PackedTree; n: NodePos): lent string =
assert n.kind == nkStrLit
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
when false:
proc strLit*(tree: PackedTree; n: NodePos): lent string =
assert n.kind == nkStrLit
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
proc strVal*(tree: PackedTree; n: NodePos): string =
assert n.kind == nkStrLit
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
#result = cookedStrLit(raw)
proc strVal*(tree: PackedTree; n: NodePos): string =
assert n.kind == nkStrLit
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
#result = cookedStrLit(raw)
proc filenameVal*(tree: PackedTree; n: NodePos): string =
case n.kind
of nkStrLit:
result = strVal(tree, n)
of nkIdent:
result = tree.sh.strings[n.litId]
of nkSym:
result = tree.sh.strings[tree.sh.syms[int n.symId].name]
else:
result = ""
proc filenameVal*(tree: PackedTree; n: NodePos): string =
case n.kind
of nkStrLit:
result = strVal(tree, n)
of nkIdent:
result = tree.sh.strings[n.litId]
of nkSym:
result = tree.sh.strings[tree.sh.syms[int n.symId].name]
else:
result = ""
proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
assert n.kind == nkIdent
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
assert n.kind == nkIdent
result = tree.sh.strings[LitId tree.nodes[n.int].operand]
const
externIntLit* = {nkCharLit,
@ -380,7 +410,8 @@ const
externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
directIntLit* = nkInt32Lit
proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int;
result: var string) =
let pos = n.int
if result.len > 0 and result[^1] notin {' ', '\n'}:
result.add ' '
@ -390,46 +421,47 @@ proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
of nkNone, nkEmpty, nkNilLit, nkType: discard
of nkIdent, nkStrLit..nkTripleStrLit:
result.add " "
result.add tree.sh.strings[LitId tree.nodes[pos].operand]
result.add sh.strings[LitId tree.nodes[pos].operand]
of nkSym:
result.add " "
result.add tree.sh.strings[tree.sh.syms[tree.nodes[pos].operand].name]
result.add sh.strings[sh.syms[tree.nodes[pos].operand].name]
of directIntLit:
result.add " "
result.addInt tree.nodes[pos].operand
of externSIntLit:
result.add " "
result.addInt tree.sh.integers[LitId tree.nodes[pos].operand]
result.addInt sh.integers[LitId tree.nodes[pos].operand]
of externUIntLit:
result.add " "
result.add $cast[uint64](tree.sh.integers[LitId tree.nodes[pos].operand])
result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand])
else:
result.add "(\n"
for i in 1..(nesting+1)*2: result.add ' '
for child in sonsReadonly(tree, n):
toString(tree, child, nesting + 1, result)
toString(tree, child, sh, nesting + 1, result)
result.add "\n"
for i in 1..nesting*2: result.add ' '
result.add ")"
#for i in 1..nesting*2: result.add ' '
proc toString*(tree: PackedTree; n: NodePos): string =
proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string =
result = ""
toString(tree, n, 0, result)
toString(tree, n, sh, 0, result)
proc debug*(tree: PackedTree) =
stdout.write toString(tree, NodePos 0)
proc debug*(tree: PackedTree; sh: Shared) =
stdout.write toString(tree, NodePos 0, sh)
proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
if n.kind == nkIdent:
result = n.litId
elif n.kind == nkSym:
result = tree.sh.syms[int n.symId].name
else:
result = LitId(0)
when false:
proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
if n.kind == nkIdent:
result = n.litId
elif n.kind == nkSym:
result = tree.sh.syms[int n.symId].name
else:
result = LitId(0)
template identId*(n: NodePos): LitId = identIdImpl(tree, n)
template identId*(n: NodePos): LitId = identIdImpl(tree, n)
template copyInto*(dest, n, body) =
let patchPos = prepare(dest, tree, n)
@ -441,17 +473,20 @@ template copyIntoKind*(dest, kind, info, body) =
body
patch dest, patchPos
proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
let litId = tree.sh.strings.getKeyId(pragma)
if litId == LitId(0):
return false
assert n.kind == nkPragma
for ch0 in sonsReadonly(tree, n):
if ch0.kind == nkExprColonExpr:
if ch0.firstSon.identId == litId:
when false:
proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
let litId = tree.sh.strings.getKeyId(pragma)
if litId == LitId(0):
return false
assert n.kind == nkPragma
for ch0 in sonsReadonly(tree, n):
if ch0.kind == nkExprColonExpr:
if ch0.firstSon.identId == litId:
return true
elif ch0.identId == litId:
return true
elif ch0.identId == litId:
return true
proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
when false:
proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
@ -459,3 +494,68 @@ when false:
dest.add nkStrLit, msg, n.info
copyTree(dest, tree, n)
patch dest, patchPos
proc hash*(table: StringTableRef): Hash =
## XXX: really should be introduced into strtabs...
var h: Hash = 0
for pair in pairs table:
h = h !& hash(pair)
result = !$h
proc hash*(config: ConfigRef): Hash =
## XXX: vet and/or extend this
var h: Hash = 0
h = h !& hash(config.selectedGC)
h = h !& hash(config.features)
h = h !& hash(config.legacyFeatures)
h = h !& hash(config.configVars)
h = h !& hash(config.symbols)
result = !$h
# XXX: lazy hashes for now
type
LazyHashes = PackedSym or PackedType or PackedLib or
PackedLineInfo or PackedTree or PackedNode
proc hash*(sh: Shared): Hash
proc hash*(s: LazyHashes): Hash
proc hash*(s: seq[LazyHashes]): Hash
proc hash*(s: LazyHashes): Hash =
var h: Hash = 0
for k, v in fieldPairs(s):
h = h !& hash((k, v))
result = !$h
proc hash*(s: seq[LazyHashes]): Hash =
## critically, we need to hash the indices alongside their values
var h: Hash = 0
for i, n in pairs s:
h = h !& hash((i, n))
result = !$h
proc hash*(sh: Shared): Hash =
## might want to edit this...
# XXX: these have too many references
when false:
var h: Hash = 0
h = h !& hash(sh.syms)
h = h !& hash(sh.types)
h = h !& hash(sh.strings)
h = h !& hash(sh.integers)
h = h !& hash(sh.floats)
h = h !& hash(sh.config)
result = !$h
proc hash*(m: Module): Hash =
var h: Hash = 0
h = h !& hash(m.name)
h = h !& hash(m.ast)
result = !$h
template safeItemId*(x: typed; f: untyped): ItemId =
## yield a valid ItemId value for the field of a nillable type
if x.isNil:
nilItemId
else:
x.`f`