diff --git a/compiler/ast.nim b/compiler/ast.nim index 8acf08284..ccf4fe497 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1074,18 +1074,35 @@ template id*(a: PIdObj): int = (x.itemId.module.int shl moduleShift) + x.itemId.item.int type - IdGenerator* = ref ItemId # unfortunately, we really need the 'shared mutable' aspect here. + IdGenerator* = ref object # unfortunately, we really need the 'shared mutable' aspect here. + module*: int32 + symId*: int32 + typeId*: int32 + +proc hash*(x: ItemId): Hash = + var h: Hash = hash(x.module) + h = h !& hash(x.item) + result = !$h const PackageModuleId* = -3'i32 proc idGeneratorFromModule*(m: PSym): IdGenerator = assert m.kind == skModule - result = IdGenerator(module: m.itemId.module, item: m.itemId.item) + result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0) -proc nextId*(x: IdGenerator): ItemId {.inline.} = - inc x.item - result = x[] +proc nextSymId*(x: IdGenerator): ItemId {.inline.} = + inc x.symId + result = ItemId(module: x.module, item: x.symId) + +proc nextTypeId*(x: IdGenerator): ItemId {.inline.} = + inc x.typeId + result = ItemId(module: x.module, item: x.typeId) + +when false: + proc nextId*(x: IdGenerator): ItemId {.inline.} = + inc x.item + result = x[] when false: proc storeBack*(dest: var IdGenerator; src: IdGenerator) {.inline.} = @@ -1831,7 +1848,7 @@ proc toVar*(typ: PType; kind: TTypeKind; idgen: IdGenerator): PType = ## returned. Otherwise ``typ`` is simply returned as-is. result = typ if typ.kind != kind: - result = newType(kind, nextId(idgen), typ.owner) + result = newType(kind, nextTypeId(idgen), typ.owner) rawAddSon(result, typ) proc toRef*(typ: PType; idgen: IdGenerator): PType = @@ -1839,7 +1856,7 @@ proc toRef*(typ: PType; idgen: IdGenerator): PType = ## returned. Otherwise ``typ`` is simply returned as-is. result = typ if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject: - result = newType(tyRef, nextId(idgen), typ.owner) + result = newType(tyRef, nextTypeId(idgen), typ.owner) rawAddSon(result, typ) proc toObject*(typ: PType): PType = diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 53b2832f0..b8a8d21b0 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1772,7 +1772,7 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: internalError(p.config, e.info, "genArrayLen()") proc makePtrType(baseType: PType; idgen: IdGenerator): PType = - result = newType(tyPtr, nextId idgen, baseType.owner) + result = newType(tyPtr, nextTypeId idgen, baseType.owner) addSonSkipIntLit(result, baseType, idgen) proc makeAddr(n: PNode; idgen: IdGenerator): PNode = diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 142fec056..ab12bad1e 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1267,9 +1267,9 @@ proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = proc fakeClosureType(m: BModule; owner: PSym): PType = # we generate the same RTTI as for a tuple[pointer, ref tuple[]] - result = newType(tyTuple, nextId m.idgen, owner) - result.rawAddSon(newType(tyPointer, nextId m.idgen, owner)) - var r = newType(tyRef, nextId m.idgen, owner) + result = newType(tyTuple, nextTypeId m.idgen, owner) + result.rawAddSon(newType(tyPointer, nextTypeId m.idgen, owner)) + var r = newType(tyRef, nextTypeId m.idgen, owner) let obj = createObj(m.g.graph, m.idgen, owner, owner.info, final=false) r.rawAddSon(obj) result.rawAddSon(r) @@ -1396,9 +1396,9 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = result = prefixTI.rope & result & ")".rope proc openArrayToTuple(m: BModule; t: PType): PType = - result = newType(tyTuple, nextId m.idgen, t.owner) - let p = newType(tyPtr, nextId m.idgen, t.owner) - let a = newType(tyUncheckedArray, nextId m.idgen, t.owner) + result = newType(tyTuple, nextTypeId m.idgen, t.owner) + let p = newType(tyPtr, nextTypeId m.idgen, t.owner) + let a = newType(tyUncheckedArray, nextTypeId m.idgen, t.owner) a.add t.lastSon p.add a result.add p diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index a0c16f2ed..5c5d35093 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -108,10 +108,10 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) = s.ast[dispatcherPos] = dispatcher proc createDispatcher(s: PSym; idgen: IdGenerator): PSym = - var disp = copySym(s, nextId(idgen)) + var disp = copySym(s, nextSymId(idgen)) incl(disp.flags, sfDispatcher) excl(disp.flags, sfExported) - disp.typ = copyType(disp.typ, nextId(idgen), disp.typ.owner) + disp.typ = copyType(disp.typ, nextTypeId(idgen), disp.typ.owner) # we can't inline the dispatcher itself (for now): if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall disp.ast = copyTree(s.ast) @@ -119,7 +119,7 @@ proc createDispatcher(s: PSym; idgen: IdGenerator): PSym = disp.loc.r = nil if s.typ[0] != nil: if disp.ast.len > resultPos: - disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextId(idgen)) + disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen)) else: # We've encountered a method prototype without a filled-in # resultPos slot. We put a placeholder in there that will diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 48088a609..43dfc69ae 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -177,7 +177,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode = ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt))) proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym = - result = newSym(skVar, getIdent(ctx.g.cache, name), nextId(ctx.idgen), ctx.fn, ctx.fn.info) + result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info) result.typ = typ assert(not typ.isNil) @@ -1118,9 +1118,9 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode= n[i] = ctx.skipThroughEmptyStates(n[i]) proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: PSym): PType = - result = newType(tyArray, nextId(idgen), owner) + result = newType(tyArray, nextTypeId(idgen), owner) - let rng = newType(tyRange, nextId(idgen), owner) + let rng = newType(tyRange, nextTypeId(idgen), owner) rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n)) rng.rawAddSon(t) @@ -1314,7 +1314,7 @@ proc freshVars(n: PNode; c: var FreshVarsContext): PNode = let idefs = copyNode(it) for v in 0..it.len-3: if it[v].kind == nkSym: - let x = copySym(it[v].sym, nextId(c.idgen)) + let x = copySym(it[v].sym, nextSymId(c.idgen)) c.tab[it[v].sym.id] = x idefs.add newSymNode(x) else: @@ -1393,9 +1393,9 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n: # Lambda lifting was not done yet. Use temporary :state sym, which will # be handled specially by lambda lifting. Local temp vars (if needed) # should follow the same logic. - ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextId(idgen), fn, fn.info) + ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextSymId(idgen), fn, fn.info) ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen) - ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextId(idgen), fn, fn.info) + ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextSymId(idgen), fn, fn.info) var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen) var n = preprocess(pc, n.toStmtList) #echo "transformed into ", n diff --git a/compiler/commands.nim b/compiler/commands.nim index 4b8755ab4..b52148668 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -31,7 +31,6 @@ import wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos, pathutils, strtabs -from incremental import nimIncremental from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect # but some have deps to imported modules. Yay. @@ -799,10 +798,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; helpOnError(conf, pass) of "symbolfiles": discard "ignore for backwards compat" of "incremental": - when not nimIncremental: - localError(conf, info, "the compiler was not built with " & - "incremental compilation features; bootstrap with " & - "-d:nimIncremental to enable") case arg.normalize of "on": conf.symbolFiles = v2Sf of "off": conf.symbolFiles = disabledSf diff --git a/compiler/enumtostr.nim b/compiler/enumtostr.nim index 3274462d7..9bfa7001a 100644 --- a/compiler/enumtostr.nim +++ b/compiler/enumtostr.nim @@ -2,15 +2,15 @@ import ast, idents, lineinfos, modulegraphs, magicsys proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym = - result = newSym(skProc, getIdent(g.cache, "$"), nextId idgen, t.owner, info) + result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info) - let dest = newSym(skParam, getIdent(g.cache, "e"), nextId idgen, result, info) + let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info) dest.typ = t - let res = newSym(skResult, getIdent(g.cache, "result"), nextId idgen, result, info) + let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info) res.typ = getSysType(g, info, tyString) - result.typ = newType(tyProc, nextId idgen, t.owner) + result.typ = newType(tyProc, nextTypeId idgen, t.owner) result.typ.n = newNodeI(nkFormalParams, info) rawAddSon(result.typ, res.typ) result.typ.n.add newNodeI(nkEffectList, info) @@ -63,15 +63,15 @@ proc searchObjCase(t: PType; field: PSym): PNode = doAssert result != nil proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym = - result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextId idgen, t.owner, info) + result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info) - let dest = newSym(skParam, getIdent(g.cache, "e"), nextId idgen, result, info) + let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info) dest.typ = field.typ - let res = newSym(skResult, getIdent(g.cache, "result"), nextId idgen, result, info) + let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info) res.typ = getSysType(g, info, tyUInt8) - result.typ = newType(tyProc, nextId idgen, t.owner) + result.typ = newType(tyProc, nextTypeId idgen, t.owner) result.typ.n = newNodeI(nkFormalParams, info) rawAddSon(result.typ, res.typ) result.typ.n.add newNodeI(nkEffectList, info) diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index 218a597d8..691d33a2c 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -49,7 +49,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = internalAssert c.config, sfGenSym in s.flags or s.kind == skType var x = PSym(idTableGet(c.mapping, s)) if x == nil: - x = copySym(s, nextId(c.idgen)) + x = copySym(s, nextSymId(c.idgen)) # sem'check needs to set the owner properly later, see bug #9476 x.owner = nil # c.genSymOwner #if x.kind == skParam and x.owner.kind == skModule: diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index 91221ea49..1f75b7759 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -1,7 +1,7 @@ ## A BiTable is a table that can be seen as an optimized pair ## of (Table[LitId, Val], Table[Val, LitId]). -import hashes +import hashes, rodfiles type LitId* = distinct uint32 @@ -30,7 +30,9 @@ proc mustRehash(length, counter: int): bool {.inline.} = result = (length * 2 < counter * 3) or (length - counter < 4) const - idStart = 256 # Ids do not start with 0 but with this value. The IR needs it. + idStart = 256 ## + ## Ids do not start with 0 but with this value. The IR needs it. + ## TODO: explain why template idToIdx(x: LitId): int = x.int - idStart @@ -94,6 +96,21 @@ proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} = assert idx < t.vals.len result = t.vals[idx] +proc hash*[T](t: BiTable[T]): Hash = + ## as the keys are hashes of the values, we simply use them instead + var h: Hash = 0 + for i, n in pairs t.keys: + h = h !& hash((i, n)) + result = !$h + +proc store*[T](f: var RodFile; t: BiTable[T]) = + storeSeq(f, t.vals) + storeSeq(f, t.keys) + +proc load*[T](f: var RodFile; t: var BiTable[T]) = + loadSeq(f, t.vals) + loadSeq(f, t.keys) + when isMainModule: var t: BiTable[string] @@ -113,7 +130,35 @@ when isMainModule: for i in 0 ..< 100_000: assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4 + echo "begin" echo t.vals.len echo t.vals[0] echo t.vals[1004] + + echo "middle" + + var tf: BiTable[float] + + discard tf.getOrIncl(0.4) + discard tf.getOrIncl(16.4) + discard tf.getOrIncl(32.4) + echo getKeyId(tf, 32.4) + + var f2 = open("testblah.bin", fmWrite) + echo store(f2, tf) + f2.close + + var f1 = open("testblah.bin", fmRead) + + var t2: BiTable[float] + + echo f1.load(t2) + echo t2.vals.len + + echo getKeyId(t2, 32.4) + + echo "end" + + + f1.close diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst index 1a33f6a27..60434e4b8 100644 --- a/compiler/ic/design.rst +++ b/compiler/ic/design.rst @@ -29,14 +29,13 @@ mechanism needs to be implemented that we could get wrong. ModuleIds are rod-file specific too. -Configuration setup changes ---------------------------- - -For a MVP these are not detected. Later the configuration will be -stored in every `.rod` file. - Global state ------------ Global persistent state will be kept in a project specific `.rod` file. + +Rod File Format +--------------- + +It's a simple binary file format. `rodfiles.nim` contains some details. diff --git a/compiler/ic/from_packed_ast.nim b/compiler/ic/from_packed_ast.nim deleted file mode 100644 index cb2ecee79..000000000 --- a/compiler/ic/from_packed_ast.nim +++ /dev/null @@ -1,12 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import std / [hashes, tables] -import bitabs -import ".." / [ast, lineinfos, options, pathutils] diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index ef609e8c8..546e495c5 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -12,9 +12,9 @@ ## use this representation directly in all the transformations, ## it is superior. -import std / [hashes, tables] +import std / [hashes, tables, strtabs, md5] import bitabs -import ".." / [ast, lineinfos, options, pathutils] +import ".." / [ast, options] const localNamePos* = 0 @@ -39,16 +39,29 @@ const routineBodyPos* = 7 const - nkModuleRef = nkNone # pair of (ModuleId, SymId) + nkModuleRef* = nkNone # pair of (ModuleId, SymId) type SymId* = distinct int32 - TypeId* = distinct int32 ModuleId* = distinct int32 NodePos* = distinct int NodeId* = distinct int32 + PackedItemId* = object + module*: LitId # 0 if it's this module + item*: int32 # same as the in-memory representation + + TypeId* = PackedItemId + +const + nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32) + nilItemId* = PackedItemId(module: LitId(0), item: -1.int32) + +const + emptyNodeId* = NodeId(-1) + +type PackedLineInfo* = object line*: uint16 col*: int16 @@ -64,13 +77,13 @@ type PackedSym* = object kind*: TSymKind name*: LitId - typeId*: TypeId + typ*: TypeId flags*: TSymFlags magic*: TMagic info*: PackedLineInfo - ast*: NodePos - owner*: ItemId - guard*: ItemId + ast*: NodeId + owner*: PackedItemId + guard*: PackedItemId bitsize*: int alignment*: int # for alignment options*: TOptions @@ -84,25 +97,25 @@ type PackedType* = object kind*: TTypeKind - nodekind*: TNodeKind + callConv*: TCallingConvention + #nodekind*: TNodeKind flags*: TTypeFlags - types*: int32 - nodes*: int32 - methods*: int32 - nodeflags*: TNodeFlags - info*: PackedLineInfo - sym*: ItemId - owner*: ItemId - attachedOps*: array[TTypeAttachedOp, ItemId] + types*: seq[TypeId] + n*: NodeId + methods*: seq[(int, PackedItemId)] + #nodeflags*: TNodeFlags + sym*: PackedItemId + owner*: PackedItemId + attachedOps*: array[TTypeAttachedOp, PackedItemId] size*: BiggestInt align*: int16 paddingAtEnd*: int16 lockLevel*: TLockLevel # lock level as required for deadlock checking # not serialized: loc*: TLoc because it is backend-specific typeInst*: TypeId - nonUniqueId*: ItemId + nonUniqueId*: int32 - Node* = object # 20 bytes + PackedNode* = object # 20 bytes kind*: TNodeKind flags*: TNodeFlags operand*: int32 # for kind in {nkSym, nkSymDef}: SymId @@ -115,72 +128,73 @@ type ModulePhase* = enum preLookup, lookedUpTopLevelStmts - Module* = object + GenericKey* = object + module*: int32 name*: string - file*: AbsoluteFile - ast*: PackedTree - phase*: ModulePhase - iface*: Table[string, seq[SymId]] # 'seq' because of overloading + types*: seq[MD5Digest] # is this a joke? - Program* = ref object - modules*: seq[Module] + PackedTree* = object ## usually represents a full Nim module + nodes*: seq[PackedNode] + #sh*: Shared Shared* = ref object # shared between different versions of 'Module'. # (though there is always exactly one valid # version of a module) syms*: seq[PackedSym] - types*: seq[seq[Node]] + types*: seq[PackedType] strings*: BiTable[string] # we could share these between modules. integers*: BiTable[BiggestInt] floats*: BiTable[BiggestFloat] - config*: ConfigRef - #thisModule*: ModuleId - #program*: Program + #config*: ConfigRef - PackedTree* = object ## usually represents a full Nim module - nodes*: seq[Node] - toPosition*: Table[SymId, NodePos] - sh*: Shared +proc hash*(key: GenericKey): Hash = + var h: Hash = 0 + h = h !& hash(key.module) + h = h !& hash(key.name) + h = h !& hash(key.types) + result = !$h proc `==`*(a, b: SymId): bool {.borrow.} proc hash*(a: SymId): Hash {.borrow.} proc `==`*(a, b: NodePos): bool {.borrow.} -proc `==`*(a, b: TypeId): bool {.borrow.} -proc `==`*(a, b: ModuleId): bool {.borrow.} - -proc declareSym*(tree: var PackedTree; kind: TSymKind; - name: LitId; info: PackedLineInfo): SymId = - result = SymId(tree.sh.syms.len) - tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info) +#proc `==`*(a, b: TypeId): bool {.borrow.} +proc `==`*(a, b: NodeId): bool {.borrow.} proc newTreeFrom*(old: PackedTree): PackedTree = result.nodes = @[] - result.sh = old.sh + when false: result.sh = old.sh -proc litIdFromName*(tree: PackedTree; name: string): LitId = - result = tree.sh.strings.getOrIncl(name) +when false: + proc declareSym*(tree: var PackedTree; kind: TSymKind; + name: LitId; info: PackedLineInfo): SymId = + result = SymId(tree.sh.syms.len) + tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info) -proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) = - tree.nodes.add Node(kind: kind, operand: int32 getOrIncl(tree.sh.strings, token), info: info) + proc litIdFromName*(tree: PackedTree; name: string): LitId = + result = tree.sh.strings.getOrIncl(name) -proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) = - tree.nodes.add Node(kind: kind, operand: 0, info: info) + proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) = + tree.nodes.add PackedNode(kind: kind, info: info, + operand: int32 getOrIncl(tree.sh.strings, token)) + + proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) = + tree.nodes.add PackedNode(kind: kind, operand: 0, info: info) proc throwAwayLastNode*(tree: var PackedTree) = tree.nodes.setLen(tree.nodes.len-1) proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) = - tree.nodes.add Node(kind: nkIdent, operand: int32(s), info: info) + tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info) -proc addSym*(tree: var PackedTree; s: SymId; info: PackedLineInfo) = - tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info) +proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) = + tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info) proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) = - tree.nodes.add Node(kind: nkInt32Lit, operand: int32(s), info: info) + tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info) proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) = - tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info) + tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info) proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit @@ -194,11 +208,12 @@ proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) = for i in 0.. nkNilLit: @@ -247,7 +264,8 @@ iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos = for x in sonsReadonly(tree, n): yield x patch dest, patchPos -iterator isons*(dest: var PackedTree; tree: PackedTree; n: NodePos): (int, NodePos) = +iterator isons*(dest: var PackedTree; tree: PackedTree; + n: NodePos): (int, NodePos) = var i = 0 for ch0 in sons(dest, tree, n): yield (i, ch0) @@ -301,10 +319,19 @@ proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool = if count >= x: return true return false -proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = NodePos(n.int+1) -proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind -proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = LitId tree.nodes[n.int].operand -proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info +proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = + NodePos(n.int+1) +proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = + tree.nodes[n.int].kind +proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = + LitId tree.nodes[n.int].operand +proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = + tree.nodes[n.int].info + +template typ*(n: NodePos): PackedItemId = + tree.nodes[n.int].typeId +template flags*(n: NodePos): TNodeFlags = + tree.nodes[n.int].flags proc span(tree: PackedTree; pos: int): int {.inline.} = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand @@ -330,7 +357,9 @@ proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos = inc count assert false, "node has no i-th child" -proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = tree.sh.strings[lit] +when false: + proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = + tree.sh.strings[lit] template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info @@ -340,29 +369,30 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1) -proc strLit*(tree: PackedTree; n: NodePos): lent string = - assert n.kind == nkStrLit - result = tree.sh.strings[LitId tree.nodes[n.int].operand] +when false: + proc strLit*(tree: PackedTree; n: NodePos): lent string = + assert n.kind == nkStrLit + result = tree.sh.strings[LitId tree.nodes[n.int].operand] -proc strVal*(tree: PackedTree; n: NodePos): string = - assert n.kind == nkStrLit - result = tree.sh.strings[LitId tree.nodes[n.int].operand] - #result = cookedStrLit(raw) + proc strVal*(tree: PackedTree; n: NodePos): string = + assert n.kind == nkStrLit + result = tree.sh.strings[LitId tree.nodes[n.int].operand] + #result = cookedStrLit(raw) -proc filenameVal*(tree: PackedTree; n: NodePos): string = - case n.kind - of nkStrLit: - result = strVal(tree, n) - of nkIdent: - result = tree.sh.strings[n.litId] - of nkSym: - result = tree.sh.strings[tree.sh.syms[int n.symId].name] - else: - result = "" + proc filenameVal*(tree: PackedTree; n: NodePos): string = + case n.kind + of nkStrLit: + result = strVal(tree, n) + of nkIdent: + result = tree.sh.strings[n.litId] + of nkSym: + result = tree.sh.strings[tree.sh.syms[int n.symId].name] + else: + result = "" -proc identAsStr*(tree: PackedTree; n: NodePos): lent string = - assert n.kind == nkIdent - result = tree.sh.strings[LitId tree.nodes[n.int].operand] + proc identAsStr*(tree: PackedTree; n: NodePos): lent string = + assert n.kind == nkIdent + result = tree.sh.strings[LitId tree.nodes[n.int].operand] const externIntLit* = {nkCharLit, @@ -380,7 +410,8 @@ const externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} directIntLit* = nkInt32Lit -proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) = +proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int; + result: var string) = let pos = n.int if result.len > 0 and result[^1] notin {' ', '\n'}: result.add ' ' @@ -390,46 +421,47 @@ proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) = of nkNone, nkEmpty, nkNilLit, nkType: discard of nkIdent, nkStrLit..nkTripleStrLit: result.add " " - result.add tree.sh.strings[LitId tree.nodes[pos].operand] + result.add sh.strings[LitId tree.nodes[pos].operand] of nkSym: result.add " " - result.add tree.sh.strings[tree.sh.syms[tree.nodes[pos].operand].name] + result.add sh.strings[sh.syms[tree.nodes[pos].operand].name] of directIntLit: result.add " " result.addInt tree.nodes[pos].operand of externSIntLit: result.add " " - result.addInt tree.sh.integers[LitId tree.nodes[pos].operand] + result.addInt sh.integers[LitId tree.nodes[pos].operand] of externUIntLit: result.add " " - result.add $cast[uint64](tree.sh.integers[LitId tree.nodes[pos].operand]) + result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand]) else: result.add "(\n" for i in 1..(nesting+1)*2: result.add ' ' for child in sonsReadonly(tree, n): - toString(tree, child, nesting + 1, result) + toString(tree, child, sh, nesting + 1, result) result.add "\n" for i in 1..nesting*2: result.add ' ' result.add ")" #for i in 1..nesting*2: result.add ' ' -proc toString*(tree: PackedTree; n: NodePos): string = +proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string = result = "" - toString(tree, n, 0, result) + toString(tree, n, sh, 0, result) -proc debug*(tree: PackedTree) = - stdout.write toString(tree, NodePos 0) +proc debug*(tree: PackedTree; sh: Shared) = + stdout.write toString(tree, NodePos 0, sh) -proc identIdImpl(tree: PackedTree; n: NodePos): LitId = - if n.kind == nkIdent: - result = n.litId - elif n.kind == nkSym: - result = tree.sh.syms[int n.symId].name - else: - result = LitId(0) +when false: + proc identIdImpl(tree: PackedTree; n: NodePos): LitId = + if n.kind == nkIdent: + result = n.litId + elif n.kind == nkSym: + result = tree.sh.syms[int n.symId].name + else: + result = LitId(0) -template identId*(n: NodePos): LitId = identIdImpl(tree, n) + template identId*(n: NodePos): LitId = identIdImpl(tree, n) template copyInto*(dest, n, body) = let patchPos = prepare(dest, tree, n) @@ -441,17 +473,20 @@ template copyIntoKind*(dest, kind, info, body) = body patch dest, patchPos -proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool = - let litId = tree.sh.strings.getKeyId(pragma) - if litId == LitId(0): - return false - assert n.kind == nkPragma - for ch0 in sonsReadonly(tree, n): - if ch0.kind == nkExprColonExpr: - if ch0.firstSon.identId == litId: +when false: + proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool = + let litId = tree.sh.strings.getKeyId(pragma) + if litId == LitId(0): + return false + assert n.kind == nkPragma + for ch0 in sonsReadonly(tree, n): + if ch0.kind == nkExprColonExpr: + if ch0.firstSon.identId == litId: + return true + elif ch0.identId == litId: return true - elif ch0.identId == litId: - return true + +proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len when false: proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) = @@ -459,3 +494,68 @@ when false: dest.add nkStrLit, msg, n.info copyTree(dest, tree, n) patch dest, patchPos + + proc hash*(table: StringTableRef): Hash = + ## XXX: really should be introduced into strtabs... + var h: Hash = 0 + for pair in pairs table: + h = h !& hash(pair) + result = !$h + + proc hash*(config: ConfigRef): Hash = + ## XXX: vet and/or extend this + var h: Hash = 0 + h = h !& hash(config.selectedGC) + h = h !& hash(config.features) + h = h !& hash(config.legacyFeatures) + h = h !& hash(config.configVars) + h = h !& hash(config.symbols) + result = !$h + + # XXX: lazy hashes for now + type + LazyHashes = PackedSym or PackedType or PackedLib or + PackedLineInfo or PackedTree or PackedNode + + proc hash*(sh: Shared): Hash + proc hash*(s: LazyHashes): Hash + proc hash*(s: seq[LazyHashes]): Hash + + proc hash*(s: LazyHashes): Hash = + var h: Hash = 0 + for k, v in fieldPairs(s): + h = h !& hash((k, v)) + result = !$h + + proc hash*(s: seq[LazyHashes]): Hash = + ## critically, we need to hash the indices alongside their values + var h: Hash = 0 + for i, n in pairs s: + h = h !& hash((i, n)) + result = !$h + + proc hash*(sh: Shared): Hash = + ## might want to edit this... + # XXX: these have too many references + when false: + var h: Hash = 0 + h = h !& hash(sh.syms) + h = h !& hash(sh.types) + h = h !& hash(sh.strings) + h = h !& hash(sh.integers) + h = h !& hash(sh.floats) + h = h !& hash(sh.config) + result = !$h + + proc hash*(m: Module): Hash = + var h: Hash = 0 + h = h !& hash(m.name) + h = h !& hash(m.ast) + result = !$h + + template safeItemId*(x: typed; f: untyped): ItemId = + ## yield a valid ItemId value for the field of a nillable type + if x.isNil: + nilItemId + else: + x.`f` diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim new file mode 100644 index 000000000..99ce183f2 --- /dev/null +++ b/compiler/ic/rodfiles.nim @@ -0,0 +1,143 @@ +# +# +# The Nim Compiler +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +from typetraits import supportsCopyMem + +type + RodSection* = enum + versionSection + configSection + stringsSection + checkSumsSection + depsSection + integersSection + floatsSection + topLevelSection + bodiesSection + symsSection + typesSection + + RodFileError* = enum + ok, tooBig, ioFailure, wrongHeader, wrongSection, configMismatch, + includeFileChanged + + RodFile* = object + f*: File + currentSection*: RodSection # for error checking + err*: RodFileError # little experiment to see if this works + # better than exceptions. + +const + RodVersion = 1 + cookie = [byte(0), byte('R'), byte('O'), byte('D'), + byte(0), byte(0), byte(0), byte(RodVersion)] + +proc storePrim*(f: var RodFile; s: string) = + if f.err != ok: return + if s.len >= high(int32): + f.err = tooBig + return + var lenPrefix = int32(s.len) + if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + f.err = ioFailure + else: + if s.len != 0: + if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: + f.err = ioFailure + +proc storePrim*[T](f: var RodFile; x: T) = + if f.err != ok: return + when supportsCopyMem(T): + if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): + f.err = ioFailure + elif T is tuple: + for y in fields(x): + storePrim(f, y) + else: + {.error: "unsupported type for 'storePrim'".} + +proc storeSeq*[T](f: var RodFile; s: seq[T]) = + if f.err != ok: return + if s.len >= high(int32): + f.err = tooBig + return + var lenPrefix = int32(s.len) + if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + f.err = ioFailure + else: + for i in 0.. 0: + if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: + f.err = ioFailure + +proc loadPrim*[T](f: var RodFile; x: var T) = + if f.err != ok: return + when supportsCopyMem(T): + if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): + f.err = ioFailure + elif T is tuple: + for y in fields(x): + loadPrim(f, y) + else: + {.error: "unsupported type for 'loadPrim'".} + +proc loadSeq*[T](f: var RodFile; s: var seq[T]) = + if f.err != ok: return + var lenPrefix = int32(0) + if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + f.err = ioFailure + else: + s = newSeq[T](lenPrefix) + for i in 0.. TypeId + symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId + config*: ConfigRef + +template primConfigFields(fn: untyped) {.dirty.} = + fn backend + fn selectedGC + fn cCompiler + fn options + fn globalOptions + +proc definedSymbolsAsString(config: ConfigRef): string = + result = newStringOfCap(200) + result.add "config" + for d in definedSymbolNames(config.symbols): + result.add ' ' + result.add d + +proc rememberConfig(c: var PackedEncoder; config: ConfigRef) = + c.m.definedSymbols = definedSymbolsAsString(config) + + template rem(x) = + c.m.cfg.x = config.x + primConfigFields rem + +proc configIdentical(m: PackedModule; config: ConfigRef): bool = + result = m.definedSymbols == definedSymbolsAsString(config) + template eq(x) = + result = result and m.cfg.x == config.x + primConfigFields eq + +proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string = + result = msgs.getHash(conf, fileIdx) + if result.len == 0: + let fullpath = msgs.toFullPath(conf, fileIdx) + result = $secureHashFile(fullpath) + msgs.setHash(conf, fileIdx, result) + +proc toLitId(x: FileIndex; c: var PackedEncoder): LitId = + ## store a file index as a literal if x == c.lastFile: result = c.lastLit else: result = c.filenames.getOrDefault(x) if result == LitId(0): - let p = msgs.toFullPath(ir.sh.config, x) - result = getOrIncl(ir.sh.strings, p) + let p = msgs.toFullPath(c.config, x) + result = getOrIncl(c.m.sh.strings, p) c.filenames[x] = result c.lastFile = x c.lastLit = result + assert result != LitId(0) -proc toPackedInfo(x: TLineInfo; ir: var PackedTree; c: var Context): PackedLineInfo = - PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, ir, c)) +proc toFileIndex(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = + result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x]) -proc toPackedType(t: PType; ir: var PackedTree; c: var Context): TypeId = - result = TypeId(0) +proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = + for it in mitems(m.includes): + if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]: + return false + result = true -proc toPackedSym(s: PSym; ir: var PackedTree; c: var Context): SymId = - result = SymId(0) +proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) = + ## setup a context for serializing to packed ast + c.m.sh = Shared() + c.thisModule = m.itemId.module + c.config = config + c.m.bodies = newTreeFrom(c.m.topLevel) + c.m.hidden = newTreeFrom(c.m.topLevel) -proc toPackedSymNode(n: PNode; ir: var PackedTree; c: var Context) = + let thisNimFile = FileIndex c.thisModule + var h = msgs.getHash(config, thisNimFile) + if h.len == 0: + let fullpath = msgs.toFullPath(config, thisNimFile) + if isAbsolute(fullpath): + # For NimScript compiler API support the main Nim file might be from a stream. + h = $secureHashFile(fullpath) + msgs.setHash(config, thisNimFile, h) + c.m.includes.add((toLitId(thisNimFile, c), h)) # the module itself + +proc addIncludeFileDep*(c: var PackedEncoder; f: FileIndex) = + c.m.includes.add((toLitId(f, c), hashFileCached(c.config, f))) + +proc addImportFileDep*(c: var PackedEncoder; f: FileIndex) = + c.m.imports.add toLitId(f, c) + +proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder) +proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId +proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId + +proc flush(c: var PackedEncoder) = + ## serialize any pending types or symbols from the context + while true: + if c.pendingTypes.len > 0: + discard toPackedType(c.pendingTypes.pop, c) + elif c.pendingSyms.len > 0: + discard toPackedSym(c.pendingSyms.pop, c) + else: + break + +proc toLitId(x: string; c: var PackedEncoder): LitId = + ## store a string as a literal + result = getOrIncl(c.m.sh.strings, x) + +proc toLitId(x: BiggestInt; c: var PackedEncoder): LitId = + ## store an integer as a literal + result = getOrIncl(c.m.sh.integers, x) + +proc toPackedInfo(x: TLineInfo; c: var PackedEncoder): PackedLineInfo = + PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c)) + +proc safeItemId(s: PSym; c: var PackedEncoder): PackedItemId {.inline.} = + ## given a symbol, produce an ItemId with the correct properties + ## for local or remote symbols, packing the symbol as necessary + if s == nil: + result = nilItemId + elif s.itemId.module == c.thisModule: + result = PackedItemId(module: LitId(0), item: s.itemId.item) + else: + result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c), + item: s.itemId.item) + +proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder) = + ## add a remote symbol reference to the tree + let info = n.info.toPackedInfo(c) + ir.nodes.add PackedNode(kind: nkModuleRef, operand: 2.int32, # 2 kids... + typeId: toPackedType(n.typ, c), info: info) + ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, + operand: toLitId(n.sym.itemId.module.FileIndex, c).int32) + ir.nodes.add PackedNode(kind: nkInt32Lit, info: info, + operand: n.sym.itemId.item) + +proc addMissing(c: var PackedEncoder; p: PSym) = + ## consider queuing a symbol for later addition to the packed tree + if p != nil and p.itemId.module == c.thisModule: + if p.itemId.item notin c.symMarker: + c.pendingSyms.add p + +proc addMissing(c: var PackedEncoder; p: PType) = + ## consider queuing a type for later addition to the packed tree + if p != nil and p.uniqueId.module == c.thisModule: + if p.uniqueId.item notin c.typeMarker: + c.pendingTypes.add p + +template storeNode(dest, src, field) = + var nodeId: NodeId + if src.field != nil: + nodeId = getNodeId(c.m.bodies) + toPackedNode(src.field, c.m.bodies, c) + else: + nodeId = emptyNodeId + dest.field = nodeId + +proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId = + ## serialize a ptype + if t.isNil: return nilTypeId + + if t.uniqueId.module != c.thisModule: + # XXX Assert here that it already was serialized in the foreign module! + # it is a foreign type: + return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c), item: t.uniqueId.item) + + if not c.typeMarker.containsOrIncl(t.uniqueId.item): + if t.uniqueId.item >= c.m.sh.types.len: + setLen c.m.sh.types, t.uniqueId.item+1 + + var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv, + size: t.size, align: t.align, nonUniqueId: t.itemId.item, + paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel) + storeNode(p, t, n) + + for op, s in pairs t.attachedOps: + c.addMissing s + p.attachedOps[op] = s.safeItemId(c) + + p.typeInst = t.typeInst.toPackedType(c) + for kid in items t.sons: + p.types.add kid.toPackedType(c) + for i, s in items t.methods: + c.addMissing s + p.methods.add (i, s.safeItemId(c)) + c.addMissing t.sym + p.sym = t.sym.safeItemId(c) + c.addMissing t.owner + p.owner = t.owner.safeItemId(c) + + # fill the reserved slot, nothing else: + c.m.sh.types[t.uniqueId.item] = p + + result = PackedItemId(module: LitId(0), item: t.uniqueId.item) + +proc toPackedLib(l: PLib; c: var PackedEncoder): PackedLib = + ## the plib hangs off the psym via the .annex field + if l.isNil: return + result.kind = l.kind + result.generated = l.generated + result.isOverriden = l.isOverriden + result.name = toLitId($l.name, c) + storeNode(result, l, path) + +proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId = + ## serialize a psym + if s.isNil: return nilItemId + + if s.itemId.module != c.thisModule: + # XXX Assert here that it already was serialized in the foreign module! + # it is a foreign symbol: + return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c), item: s.itemId.item) + + if not c.symMarker.containsOrIncl(s.itemId.item): + if s.itemId.item >= c.m.sh.syms.len: + setLen c.m.sh.syms, s.itemId.item+1 + + var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c), magic: s.magic, + position: s.position, offset: s.offset, options: s.options, + name: s.name.s.toLitId(c)) + + storeNode(p, s, ast) + storeNode(p, s, constraint) + + if s.kind in {skLet, skVar, skField, skForVar}: + c.addMissing s.guard + p.guard = s.guard.safeItemId(c) + p.bitsize = s.bitsize + p.alignment = s.alignment + + p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, c) + c.addMissing s.typ + p.typ = s.typ.toPackedType(c) + c.addMissing s.owner + p.owner = s.owner.safeItemId(c) + p.annex = toPackedLib(s.annex, c) + when hasFFI: + p.cname = toLitId(s.cname, c) + + # fill the reserved slot, nothing else: + c.m.sh.syms[s.itemId.item] = p + + result = PackedItemId(module: LitId(0), item: s.itemId.item) + +proc toSymNode(n: PNode; ir: var PackedTree; c: var PackedEncoder) = + ## store a local or remote psym reference in the tree assert n.kind == nkSym - let t = toPackedType(n.typ, ir, c) - - if n.sym.itemId.module == c.thisModule: + template s: PSym = n.sym + let id = s.toPackedSym(c).item + if s.itemId.module == c.thisModule: # it is a symbol that belongs to the module we're currently # packing: - let sid = toPackedSym(n.sym, ir, c) - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(sid), - typeId: t, info: toPackedInfo(n.info, ir, c)) + ir.addSym(id, toPackedInfo(n.info, c)) else: # store it as an external module reference: - # nkModuleRef - discard - - -proc toPackedNode*(n: PNode; ir: var PackedTree; c: var Context) = - template toP(x: TLineInfo): PackedLineInfo = toPackedInfo(x, ir, c) + addModuleRef(n, ir, c) +proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder) = + ## serialize a node into the tree + if n.isNil: return + let info = toPackedInfo(n.info, c) case n.kind - of nkNone, nkEmpty, nkNilLit: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: 0, - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + of nkNone, nkEmpty, nkNilLit, nkType: + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0, + typeId: toPackedType(n.typ, c), info: info) of nkIdent: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.ident.s), - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, + operand: int32 getOrIncl(c.m.sh.strings, n.ident.s), + typeId: toPackedType(n.typ, c), info: info) of nkSym: - toPackedSymNode(n, ir, c) + toSymNode(n, ir, c) of directIntLit: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(n.intVal), - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, + operand: int32(n.intVal), + typeId: toPackedType(n.typ, c), info: info) of externIntLit: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.integers, n.intVal), - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, + operand: int32 getOrIncl(c.m.sh.integers, n.intVal), + typeId: toPackedType(n.typ, c), info: info) of nkStrLit..nkTripleStrLit: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.strVal), - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, + operand: int32 getOrIncl(c.m.sh.strings, n.strVal), + typeId: toPackedType(n.typ, c), info: info) of nkFloatLit..nkFloat128Lit: - ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.floats, n.floatVal), - typeId: toPackedType(n.typ, ir, c), info: toP n.info) + ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, + operand: int32 getOrIncl(c.m.sh.floats, n.floatVal), + typeId: toPackedType(n.typ, c), info: info) else: - let patchPos = ir.prepare(n.kind, n.flags, toPackedType(n.typ, ir, c), toP n.info) + let patchPos = ir.prepare(n.kind, n.flags, + toPackedType(n.typ, c), info) for i in 0..= g.len: + g.setLen(m+1) + + case g[m].status + of undefined: + g[m].status = loading + let fullpath = msgs.toFullPath(conf, fileIdx) + let rod = toRodFile(conf, AbsoluteFile fullpath) + let err = loadRodFile(rod, g[m].fromDisk, conf) + if err == ok: + result = false + # check its dependencies: + for dep in g[m].fromDisk.imports: + let fid = toFileIndex(dep, g[m].fromDisk, conf) + # Warning: we need to traverse the full graph, so + # do **not use break here**! + if needsRecompile(g, conf, fid): + result = true + + g[m].status = if result: outdated else: loaded + else: + loadError(err, rod) + g[m].status = outdated + result = true + of loading, loaded: + result = false + of outdated: + result = true + +# ------------------------------------------------------------------------- + +proc storeError(err: RodFileError; filename: AbsoluteFile) = + echo "Error: ", $err, "; couldn't write to ", filename.string + removeFile(filename.string) + +proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) = + rememberConfig(encoder, encoder.config) + + var f = rodfiles.create(filename.string) + f.storeHeader() + f.storeSection configSection + f.storePrim encoder.m.definedSymbols + f.storePrim encoder.m.cfg + + f.storeSection stringsSection + f.store encoder.m.sh.strings + + f.storeSection checkSumsSection + f.storeSeq encoder.m.includes + + f.storeSection depsSection + f.storeSeq encoder.m.imports + + f.storeSection integersSection + f.store encoder.m.sh.integers + + f.storeSection floatsSection + f.store encoder.m.sh.floats + + f.storeSection topLevelSection + f.storeSeq encoder.m.topLevel.nodes + + f.storeSection bodiesSection + f.storeSeq encoder.m.bodies.nodes + + f.storeSection symsSection + f.storeSeq encoder.m.sh.syms + + f.storeSection typesSection + f.storeSeq encoder.m.sh.types + close(f) + if f.err != ok: + loadError(f.err, filename) + + when true: + # basic loader testing: + var m2: PackedModule + discard loadRodFile(filename, m2, encoder.config) + +# ---------------------------------------------------------------------------- + +type + PackedDecoder* = object + thisModule*: int32 + lastLit*: LitId + lastFile*: FileIndex # remember the last lookup entry. + config*: ConfigRef + ident: IdentCache + +proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType +proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym + +proc toFileIndexCached(c: var PackedDecoder; g: var PackedModuleGraph; f: LitId): FileIndex = + if c.lastLit == f: + result = c.lastFile + else: + result = toFileIndex(f, g[c.thisModule].fromDisk, c.config) + c.lastLit = f + c.lastFile = result + +proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph; + x: PackedLineInfo): TLineInfo = + assert g[c.thisModule].status == loaded + result = TLineInfo(line: x.line, col: x.col, + fileIndex: toFileIndexCached(c, g, x.file)) + +proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph; + tree: PackedTree; n: NodePos): PNode = + let k = n.kind + result = newNodeIT(k, translateLineInfo(c, g, n.info), + loadType(c, g, n.typ)) + result.flags = n.flags + + case k + of nkEmpty, nkNilLit, nkType: + discard + of nkIdent: + result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId]) + of nkSym: + result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand)) + of directIntLit: + result.intVal = tree.nodes[n.int].operand + of externIntLit: + result.intVal = g[c.thisModule].fromDisk.sh.integers[n.litId] + of nkStrLit..nkTripleStrLit: + result.strVal = g[c.thisModule].fromDisk.sh.strings[n.litId] + of nkFloatLit..nkFloat128Lit: + result.floatVal = g[c.thisModule].fromDisk.sh.floats[n.litId] + of nkModuleRef: + let (n1, n2) = sons2(tree, n) + assert n1.kind == nkInt32Lit + assert n2.kind == nkInt32Lit + transitionNoneToSym(result) + result.sym = loadSym(c, g, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand)) + else: + for n0 in sonsReadonly(tree, n): + result.add loadNodes(c, g, tree, n0) + +proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph; + s: PackedItemId): int32 {.inline.} = + result = if s.module == LitId(0): c.thisModule + else: toFileIndexCached(c, g, s.module).int32 + +proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + s: PackedSym; si, item: int32): PSym = + result = PSym(itemId: ItemId(module: si, item: item), + kind: s.kind, magic: s.magic, flags: s.flags, + info: translateLineInfo(c, g, s.info), + options: s.options, + position: s.position, + name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name]) + ) + +template loadAstBody(p, field) = + if p.field != emptyNodeId: + result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field) + +proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph; + si, item: int32; l: PackedLib): PLib = + # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty) + if l.name.int == 0: + result = nil + else: + result = PLib(generated: l.generated, isOverriden: l.isOverriden, + kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name]) + loadAstBody(l, path) + +proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + s: PackedSym; si, item: int32; result: PSym) = + result.typ = loadType(c, g, s.typ) + loadAstBody(s, constraint) + loadAstBody(s, ast) + result.annex = loadLib(c, g, si, item, s.annex) + when hasFFI: + result.cname = g[si].fromDisk.sh.strings[s.cname] + + if s.kind in {skLet, skVar, skField, skForVar}: + result.guard = loadSym(c, g, s.guard) + result.bitsize = s.bitsize + result.alignment = s.alignment + result.owner = loadSym(c, g, s.owner) + let externalName = g[si].fromDisk.sh.strings[s.externalName] + if externalName != "": + result.loc.r = rope externalName + +proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym = + if s == nilTypeId: + result = nil + else: + let si = moduleIndex(c, g, s) + assert g[si].status == loaded + if not g[si].symsInit: + g[si].symsInit = true + setLen g[si].syms, g[si].fromDisk.sh.syms.len + + if g[si].syms[s.item] == nil: + let packed = addr(g[si].fromDisk.sh.syms[s.item]) + result = symHeaderFromPacked(c, g, packed[], si, s.item) + # store it here early on, so that recursions work properly: + g[si].syms[s.item] = result + symBodyFromPacked(c, g, packed[], si, s.item, result) + else: + result = g[si].syms[s.item] + +proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + t: PackedType; si, item: int32): PType = + result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind, + flags: t.flags, size: t.size, align: t.align, + paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel, + uniqueId: ItemId(module: si, item: item)) + +proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; + t: PackedType; si, item: int32; result: PType) = + result.sym = loadSym(c, g, t.sym) + result.owner = loadSym(c, g, t.owner) + for op, item in pairs t.attachedOps: + result.attachedOps[op] = loadSym(c, g, item) + result.typeInst = loadType(c, g, t.typeInst) + for son in items t.types: + result.sons.add loadType(c, g, son) + loadAstBody(t, n) + for gen, id in items t.methods: + result.methods.add((gen, loadSym(c, g, id))) + +proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType = + if t == nilTypeId: + result = nil + else: + let si = moduleIndex(c, g, t) + assert g[si].status == loaded + if not g[si].typesInit: + g[si].typesInit = true + setLen g[si].types, g[si].fromDisk.sh.types.len + + if g[si].types[t.item] == nil: + let packed = addr(g[si].fromDisk.sh.types[t.item]) + result = typeHeaderFromPacked(c, g, packed[], si, t.item) + # store it here early on, so that recursions work properly: + g[si].types[t.item] = result + typeBodyFromPacked(c, g, packed[], si, t.item, result) + else: + result = g[si].types[t.item] + + +when false: + proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey = + result.module = s.owner.itemId.module + result.name = s.name.s + result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest + + proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) = + ## add a generic to the module + if key notin m.generics: + m.generics[key] = toPackedSym(s, m.ast, c) + toPackedNode(s.ast, m.ast, c) diff --git a/compiler/importer.nim b/compiler/importer.nim index a055e16b7..645c03b2b 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -220,12 +220,13 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym = localError(c.config, n.info, "module alias must be an identifier") elif n[1].ident.id != realModule.name.id: # some misguided guy will write 'import abc.foo as foo' ... - result = createModuleAlias(realModule, nextId c.idgen, n[1].ident, realModule.info, + result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info, c.config.options) proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym = let f = checkModuleName(c.config, n) if f != InvalidFileIdx: + addImportFileDep(c, f) let L = c.graph.importStack.len let recursion = c.graph.importStack.find(f) c.graph.importStack.add f diff --git a/compiler/incremental.nim b/compiler/incremental.nim deleted file mode 100644 index 8b3a9bf55..000000000 --- a/compiler/incremental.nim +++ /dev/null @@ -1,198 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2018 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Basic type definitions the module graph needs in order to support -## incremental compilations. - -const nimIncremental* = defined(nimIncremental) - -import options, lineinfos - -when nimIncremental: - import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils - from strutils import parseInt - from os import isAbsolute - - type - Writer* = object - sstack*: seq[PSym] # a stack of symbols to process - tstack*: seq[PType] # a stack of types to process - tmarks*, smarks*: IntSet - forwardedSyms*: seq[PSym] - - Reader* = object - syms*: BTree[int, PSym] - types*: BTree[int, PType] - - IncrementalCtx* = object - db*: DbConn - w*: Writer - r*: Reader - configChanged*: bool - - proc init*(incr: var IncrementalCtx) = - incr.w.sstack = @[] - incr.w.tstack = @[] - incr.w.tmarks = initIntSet() - incr.w.smarks = initIntSet() - incr.w.forwardedSyms = @[] - incr.r.syms = initBTree[int, PSym]() - incr.r.types = initBTree[int, PType]() - - - proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string = - result = msgs.getHash(conf, fileIdx) - if result.len == 0 and isAbsolute(string fullpath): - result = $secureHashFile(string fullpath) - msgs.setHash(conf, fileIdx, result) - - proc toDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; fileIdx: FileIndex): int = - if fileIdx == FileIndex(-1): return -1 - let fullpath = toFullPath(conf, fileIdx) - let row = incr.db.getRow(sql"select id, fullhash from filenames where fullpath = ?", - fullpath) - let id = row[0] - let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath) - if id.len == 0: - result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)", - int(fileIdx), fullpath, fullhash) - else: - if row[1] != fullhash: - incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath) - result = parseInt(id) - - proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex = - if dbId == -1: return FileIndex(-1) - let fullpath = incr.db.getValue(sql"select fullpath from filenames where id = ?", dbId) - doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId - result = fileInfoIdx(conf, AbsoluteFile fullpath) - - - proc addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef; - module, fileIdx: FileIndex; - isIncludeFile: bool) = - if conf.symbolFiles != v2Sf: return - - let a = toDbFileId(incr, conf, module) - let b = toDbFileId(incr, conf, fileIdx) - - incr.db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)", - a, b, ord(isIncludeFile)) - - # --------------- Database model --------------------------------------------- - - proc createDb*(db: DbConn) = - db.exec(sql""" - create table if not exists controlblock( - idgen integer not null - ); - """) - - db.exec(sql""" - create table if not exists config( - config varchar(8000) not null - ); - """) - - db.exec(sql""" - create table if not exists filenames( - id integer primary key, - nimid integer not null, - fullpath varchar(8000) not null, - fullHash varchar(256) not null - ); - """) - db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" - - db.exec(sql""" - create table if not exists modules( - id integer primary key, - nimid integer not null, - fullpath varchar(8000) not null, - interfHash varchar(256) not null, - fullHash varchar(256) not null, - - created timestamp not null default (DATETIME('now')) - );""") - db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""") - - db.exec(sql""" - create table if not exists deps( - id integer primary key, - module integer not null, - dependency integer not null, - isIncludeFile integer not null, - foreign key (module) references filenames(id), - foreign key (dependency) references filenames(id) - );""") - db.exec(sql"""create index if not exists DepsIx on deps(module);""") - - db.exec(sql""" - create table if not exists types( - id integer primary key, - nimid integer not null, - module integer not null, - data blob not null, - foreign key (module) references module(id) - ); - """) - db.exec sql"create index TypeByModuleIdx on types(module);" - db.exec sql"create index TypeByNimIdIdx on types(nimid);" - - db.exec(sql""" - create table if not exists syms( - id integer primary key, - nimid integer not null, - module integer not null, - name varchar(256) not null, - data blob not null, - exported int not null, - foreign key (module) references module(id) - ); - """) - db.exec sql"create index if not exists SymNameIx on syms(name);" - db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);" - db.exec sql"create index SymByModuleIdx on syms(module);" - db.exec sql"create index SymByNimIdIdx on syms(nimid);" - - - db.exec(sql""" - create table if not exists toplevelstmts( - id integer primary key, - position integer not null, - module integer not null, - data blob not null, - foreign key (module) references module(id) - ); - """) - db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);" - db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);" - - db.exec(sql""" - create table if not exists statics( - id integer primary key, - module integer not null, - data blob not null, - foreign key (module) references module(id) - ); - """) - db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);" - db.exec sql"insert into controlblock(idgen) values (0)" - - -else: - type - IncrementalCtx* = object - - template init*(incr: IncrementalCtx) = discard - - template addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef; - module, fileIdx: FileIndex; - isIncludeFile: bool) = - discard diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 34c11e06c..76a063faa 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -62,7 +62,7 @@ template dbg(body) = body proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode = - let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextId c.idgen, c.owner, info) + let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info) sym.typ = typ s.vars.add(sym) result = newSymNode(sym) @@ -252,7 +252,7 @@ proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) = localError(c.graph.config, ri.info, errGenerated, m) proc makePtrType(c: var Con, baseType: PType): PType = - result = newType(tyPtr, nextId c.idgen, c.owner) + result = newType(tyPtr, nextTypeId c.idgen, c.owner) addSonSkipIntLit(result, baseType, c.idgen) proc genOp(c: var Con; op: PSym; dest: PNode): PNode = @@ -415,7 +415,7 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode = else: result = newNodeIT(nkStmtListExpr, n.info, n.typ) - var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextId c.idgen, c.owner, n.info) + var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextSymId c.idgen, c.owner, n.info) temp.typ = n.typ var v = newNodeI(nkLetSection, n.info) let tempAsNode = newSymNode(temp) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 4f3823f8a..8eaa9e1e2 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -130,14 +130,14 @@ proc createClosureIterStateType*(g: ModuleGraph; iter: PSym; idgen: IdGenerator) var n = newNodeI(nkRange, iter.info) n.add newIntNode(nkIntLit, -1) n.add newIntNode(nkIntLit, 0) - result = newType(tyRange, nextId(idgen), iter) + result = newType(tyRange, nextTypeId(idgen), iter) result.n = n var intType = nilOrSysInt(g) - if intType.isNil: intType = newType(tyInt, nextId(idgen), iter) + if intType.isNil: intType = newType(tyInt, nextTypeId(idgen), iter) rawAddSon(result, intType) proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym = - result = newSym(skField, getIdent(g.cache, ":state"), nextId(idgen), iter, iter.info) + result = newSym(skField, getIdent(g.cache, ":state"), nextSymId(idgen), iter, iter.info) result.typ = createClosureIterStateType(g, iter, idgen) proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType = @@ -151,7 +151,7 @@ proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym result = iter.ast[resultPos].sym else: # XXX a bit hacky: - result = newSym(skResult, getIdent(g.cache, ":result"), nextId(idgen), iter, iter.info, {}) + result = newSym(skResult, getIdent(g.cache, ":result"), nextSymId(idgen), iter, iter.info, {}) result.typ = iter.typ[0] incl(result.flags, sfUsed) iter.ast.add newSymNode(result) @@ -259,7 +259,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache, idgen) env = indirectAccess(newSymNode(it), hp, hp.info) else: - let e = newSym(skLet, iter.name, nextId(idgen), owner, n.info) + let e = newSym(skLet, iter.name, nextSymId(idgen), owner, n.info) e.typ = hp.typ e.flags = hp.flags env = newSymNode(e) @@ -330,7 +330,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym; info: TLineInfo): PType = result = c.ownerToType.getOrDefault(owner.id) if result.isNil: - result = newType(tyRef, nextId(c.idgen), owner) + result = newType(tyRef, nextTypeId(c.idgen), owner) let obj = createEnvObj(c.graph, c.idgen, owner, info) rawAddSon(result, obj) c.ownerToType[owner.id] = result @@ -338,7 +338,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym; proc asOwnedRef(c: var DetectionPass; t: PType): PType = if optOwnedRefs in c.graph.config.globalOptions: assert t.kind == tyRef - result = newType(tyOwned, nextId(c.idgen), t.owner) + result = newType(tyOwned, nextTypeId(c.idgen), t.owner) result.flags.incl tfHasOwned result.rawAddSon t else: @@ -347,7 +347,7 @@ proc asOwnedRef(c: var DetectionPass; t: PType): PType = proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym; info: TLineInfo): PType = var r = c.getEnvTypeForOwner(owner, info) - result = newType(tyPtr, nextId(c.idgen), owner) + result = newType(tyPtr, nextTypeId(c.idgen), owner) rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr})) proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) = @@ -375,7 +375,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) = if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags: localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor") else: - let result = newSym(skField, upIdent, nextId(c.idgen), obj.owner, obj.owner.info) + let result = newSym(skField, upIdent, nextSymId(c.idgen), obj.owner, obj.owner.info) result.typ = fieldType when false: if c.graph.config.selectedGC == gcDestructors: @@ -413,7 +413,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) = let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner let t = c.getEnvTypeForOwner(owner, info) if cp == nil: - cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextId(c.idgen), fn, fn.info) + cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextSymId(c.idgen), fn, fn.info) incl(cp.flags, sfFromGeneric) cp.typ = t addHiddenParam(fn, cp) @@ -539,7 +539,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode = result = n proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo; idgen: IdGenerator): PNode = - var v = newSym(skVar, getIdent(cache, envName), nextId(idgen), owner, info) + var v = newSym(skVar, getIdent(cache, envName), nextSymId(idgen), owner, info) v.flags = {sfShadowed, sfGeneratedOp} v.typ = typ result = newSymNode(v) @@ -563,7 +563,7 @@ proc setupEnvVar(owner: PSym; d: var DetectionPass; result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info, d.idgen) c.envVars[owner.id] = result if optOwnedRefs in d.graph.config.globalOptions: - var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextId d.idgen, owner, info) + var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextSymId d.idgen, owner, info) v.flags = {sfShadowed, sfGeneratedOp} v.typ = envVarType c.unownedEnvVars[owner.id] = newSymNode(v) @@ -647,7 +647,7 @@ proc closureCreationForIter(iter: PNode; d: var DetectionPass; c: var LiftingPass): PNode = result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ) let owner = iter.sym.skipGenericOwner - var v = newSym(skVar, getIdent(d.graph.cache, envName), nextId(d.idgen), owner, iter.info) + var v = newSym(skVar, getIdent(d.graph.cache, envName), nextSymId(d.idgen), owner, iter.info) incl(v.flags, sfShadowed) v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ) var vnode: PNode @@ -937,7 +937,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym): let iter = op.sym let hp = getHiddenParam(g, iter) - env = newSym(skLet, iter.name, nextId(idgen), owner, body.info) + env = newSym(skLet, iter.name, nextSymId(idgen), owner, body.info) env.typ = hp.typ env.flags = hp.flags diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 512fb6190..5ffa25e53 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -222,7 +222,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) = # for every field (dependent on dest.kind): # `=` dest.field, src.field # =destroy(blob) - var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId c.idgen, c.fn, c.info) + var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info) temp.typ = x.typ incl(temp.flags, sfFromGeneric) var v = newNodeI(nkVarSection, c.info) @@ -317,7 +317,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode; if optSeqDestructors in c.g.config.globalOptions: var op = field let destructorOverriden = destructorOverriden(t) - if op != nil and op != c.fn and + if op != nil and op != c.fn and (sfOverriden in op.flags or destructorOverriden): if sfError in op.flags: incl c.fn.flags, sfError @@ -412,7 +412,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = result = true proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode = - var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId(c.idgen), c.fn, c.info) + var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info) temp.typ = getSysType(c.g, body.info, tyInt) incl(temp.flags, sfFromGeneric) @@ -422,7 +422,7 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode = body.add v proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode = - var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId(c.idgen), c.fn, c.info) + var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info) temp.typ = value.typ incl(temp.flags, sfFromGeneric) @@ -899,17 +899,17 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp info: TLineInfo; idgen: IdGenerator): PSym = let procname = getIdent(g.cache, AttachedOpToStr[kind]) - result = newSym(skProc, procname, nextId(idgen), owner, info) - let dest = newSym(skParam, getIdent(g.cache, "dest"), nextId(idgen), result, info) + result = newSym(skProc, procname, nextSymId(idgen), owner, info) + let dest = newSym(skParam, getIdent(g.cache, "dest"), nextSymId(idgen), result, info) let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), - nextId(idgen), result, info) + nextSymId(idgen), result, info) dest.typ = makeVarType(typ.owner, typ, idgen) if kind == attachedTrace: src.typ = getSysType(g, info, tyPointer) else: src.typ = typ - result.typ = newProcType(info, nextId(idgen), owner) + result.typ = newProcType(info, nextTypeId(idgen), owner) result.typ.addParam dest if kind notin {attachedDestructor, attachedDispose}: result.typ.addParam src @@ -917,7 +917,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp if kind == attachedAsgn and g.config.selectedGC == gcOrc and cyclicType(typ.skipTypes(abstractInst)): let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"), - nextId(idgen), result, info) + nextSymId(idgen), result, info) cycleParam.typ = getSysType(g, info, tyBool) result.typ.addParam cycleParam @@ -983,7 +983,7 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, a.addMemReset = true let discrimantDest = result.typ.n[1].sym - let dst = newSym(skVar, getIdent(g.cache, "dest"), nextId(idgen), result, info) + let dst = newSym(skVar, getIdent(g.cache, "dest"), nextSymId(idgen), result, info) dst.typ = makePtrType(typ.owner, typ, idgen) let dstSym = newSymNode(dst) let d = newDeref(dstSym) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index fba032c1e..3c0aabe9a 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -227,7 +227,7 @@ proc errorSym*(c: PContext, n: PNode): PSym = considerQuotedIdent(c, m) else: getIdent(c.cache, "err:" & renderTree(m)) - result = newSym(skError, ident, nextId(c.idgen), getCurrOwner(c), n.info, {}) + result = newSym(skError, ident, nextSymId(c.idgen), getCurrOwner(c), n.info, {}) result.typ = errorType(c) incl(result.flags, sfDiscardable) # pretend it's from the top level scope to prevent cascading errors: diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index e9e704075..37405d8d9 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -71,7 +71,7 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P let value = n.lastSon result = newNodeI(nkStmtList, n.info) - var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextId(idgen), + var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, value.info, g.config.options) temp.typ = skipTypes(value.typ, abstractInst) incl(temp.flags, sfFromGeneric) @@ -91,7 +91,7 @@ proc evalOnce*(g: ModuleGraph; value: PNode; idgen: IdGenerator; owner: PSym): P ## freely, multiple times. This is frequently required and such a builtin would also be ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'! result = newNodeIT(nkStmtListExpr, value.info, value.typ) - var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextId(idgen), + var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, value.info, g.config.options) temp.typ = skipTypes(value.typ, abstractInst) incl(temp.flags, sfFromGeneric) @@ -117,7 +117,7 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o let value = n.lastSon result = newNodeI(nkStmtList, n.info) - var temp = newSym(skTemp, getIdent(g.cache, "_"), nextId(idgen), owner, value.info, owner.options) + var temp = newSym(skTemp, getIdent(g.cache, "_"), nextSymId(idgen), owner, value.info, owner.options) var v = newNodeI(nkLetSection, value.info) let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info) @@ -135,7 +135,7 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode = result = newNodeI(nkStmtList, n.info) # note: cannot use 'skTemp' here cause we really need the copy for the VM :-( - var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextId(idgen), owner, n.info, owner.options) + var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, n.info, owner.options) temp.typ = n[1].typ incl(temp.flags, sfFromGeneric) incl(temp.flags, sfGenSym) @@ -154,7 +154,7 @@ proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNod result.add newFastAsgnStmt(n[2], tempAsNode) proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo; final=true): PType = - result = newType(tyObject, nextId(idgen), owner) + result = newType(tyObject, nextTypeId(idgen), owner) if final: rawAddSon(result, nil) incl result.flags, tfFinal @@ -162,7 +162,7 @@ proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo rawAddSon(result, getCompilerProc(g, "RootObj").typ) result.n = newNodeI(nkRecList, info) let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s), - nextId(idgen), + nextSymId(idgen), owner, info, owner.options) incl s.flags, sfAnon s.typ = result @@ -225,7 +225,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = # because of 'gensym' support, we have to mangle the name with its ID. # This is hacky but the clean solution is much more complex than it looks. var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), - nextId(idgen), s.owner, s.info, s.options) + nextSymId(idgen), s.owner, s.info, s.options) field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item) let t = skipIntLit(s.typ, idgen) field.typ = t @@ -239,7 +239,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} = result = lookupInRecord(obj.n, s.itemId) if result == nil: - var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextId(idgen), + var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextSymId(idgen), s.owner, s.info, s.options) field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item) let t = skipIntLit(s.typ, idgen) @@ -326,7 +326,7 @@ proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode = proc genAddrOf*(n: PNode; idgen: IdGenerator; typeKind = tyPtr): PNode = result = newNodeI(nkAddr, n.info, 1) result[0] = n - result.typ = newType(typeKind, nextId(idgen), n.typ.owner) + result.typ = newType(typeKind, nextTypeId(idgen), n.typ.owner) result.typ.rawAddSon(n.typ) proc genDeref*(n: PNode; k = nkHiddenDeref): PNode = diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 71003371e..d700ab9a7 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -21,7 +21,7 @@ proc registerSysType*(g: ModuleGraph; t: PType) = if g.sysTypes[t.kind] == nil: g.sysTypes[t.kind] = t proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType = - result = newType(kind, nextId(g.idgen), g.systemModule) + result = newType(kind, nextTypeId(g.idgen), g.systemModule) result.size = size result.align = size.int16 @@ -29,8 +29,8 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym = result = strTableGet(g.systemModule.tab, getIdent(g.cache, name)) if result == nil: localError(g.config, info, "system module needs: " & name) - result = newSym(skError, getIdent(g.cache, name), nextId(g.idgen), g.systemModule, g.systemModule.info, {}) - result.typ = newType(tyError, nextId(g.idgen), g.systemModule) + result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {}) + result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule) if result.kind == skAlias: result = result.owner proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym = @@ -45,8 +45,8 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy r = nextIdentIter(ti, g.systemModule.tab) if result != nil: return result localError(g.config, info, "system module needs: " & name) - result = newSym(skError, id, nextId(g.idgen), g.systemModule, g.systemModule.info, {}) - result.typ = newType(tyError, nextId(g.idgen), g.systemModule) + result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {}) + result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule) proc sysTypeFromName*(g: ModuleGraph; info: TLineInfo; name: string): PType = result = getSysSym(g, info, name).typ @@ -101,12 +101,12 @@ proc getIntLitType*(g: ModuleGraph; literal: PNode): PType = result = g.intTypeCache[value.int] if result == nil: let ti = getSysType(g, literal.info, tyInt) - result = copyType(ti, nextId(g.idgen), ti.owner) + result = copyType(ti, nextTypeId(g.idgen), ti.owner) result.n = literal g.intTypeCache[value.int] = result else: let ti = getSysType(g, literal.info, tyInt) - result = copyType(ti, nextId(g.idgen), ti.owner) + result = copyType(ti, nextTypeId(g.idgen), ti.owner) result.n = literal proc getFloatLitType*(g: ModuleGraph; literal: PNode): PType = @@ -116,7 +116,7 @@ proc getFloatLitType*(g: ModuleGraph; literal: PNode): PType = proc skipIntLit*(t: PType; id: IdGenerator): PType {.inline.} = if t.n != nil and t.kind in {tyInt, tyFloat}: - result = copyType(t, nextId(id), t.owner) + result = copyType(t, nextTypeId(id), t.owner) result.n = nil else: result = t diff --git a/compiler/main.nim b/compiler/main.nim index c94c4323f..868198268 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -19,7 +19,7 @@ import cgen, json, nversion, platform, nimconf, passaux, depends, vm, modules, - modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler + modulegraphs, tables, lineinfos, pathutils, vmprofiler when not defined(leanCompiler): import jsgen, docgen, docgen2 @@ -137,7 +137,7 @@ proc commandInteractive(graph: ModuleGraph) = else: var m = graph.makeStdinModule() incl(m.flags, sfMainModule) - var idgen = IdGenerator(module: m.itemId.module, item: m.itemId.item) + var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0) let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config)) processModule(graph, m, idgen, s) @@ -165,7 +165,6 @@ proc mainCommand*(graph: ModuleGraph) = let conf = graph.config let cache = graph.cache - setupModuleCache(graph) # In "nim serve" scenario, each command must reset the registered passes clearPasses(graph) conf.lastCmdTime = epochTime() diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index e3c54eeae..5544668cc 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -9,7 +9,7 @@ ## This module implements the module graph data structure. The module graph ## represents a complete Nim project. Single modules can either be kept in RAM -## or stored in a Sqlite database. +## or stored in a rod-file. ## ## The caching of modules is critical for 'nimsuggest' and is tricky to get ## right. If module E is being edited, we need autocompletion (and type @@ -26,7 +26,7 @@ ## import ast, intsets, tables, options, lineinfos, hashes, idents, - incremental, btrees, md5 + btrees, md5 # import ic / packed_ast @@ -67,7 +67,6 @@ type intTypeCache*: array[-5..64, PType] opContains*, opNot*: PSym emptyNode*: PNode - incr*: IncrementalCtx canonTypes*: Table[SigHash, PType] symBodyHashes*: Table[int, SigHash] # symId to digest mapping importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.} @@ -176,7 +175,7 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} = result = g.doStopCompile != nil and g.doStopCompile() proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym = - result = newSym(skProc, getIdent(g.cache, name), nextId(g.idgen), nil, unknownLineInfo, {}) + result = newSym(skProc, getIdent(g.cache, name), nextSymId(g.idgen), nil, unknownLineInfo, {}) result.magic = m result.flags = {sfNeverRaises} @@ -190,7 +189,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) = proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result = ModuleGraph() - result.idgen = IdGenerator(module: -1'i32, item: 0'i32) + result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32) initStrTable(result.packageSyms) result.deps = initIntSet() result.importDeps = initTable[FileIndex, seq[FileIndex]]() @@ -206,7 +205,6 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.opNot = createMagic(result, "not", mNot) result.opContains = createMagic(result, "contains", mInSet) result.emptyNode = newNode(nkEmpty) - init(result.incr) result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} = discard result.cacheSeqs = initTable[string, PNode]() @@ -235,7 +233,6 @@ proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = assert m.position == m.info.fileIndex.int32 - addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false) if g.suggestMode: g.deps.incl m.position.dependsOn(dep.int) # we compute the transitive closure later when querying the graph lazily. @@ -243,7 +240,6 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) = #invalidTransitiveClosure = true proc addIncludeDep*(g: ModuleGraph; module, includeFile: FileIndex) = - addModuleDep(g.incr, g.config, module, includeFile, isIncludeFile = true) discard hasKeyOrPut(g.inclToMod, includeFile, module) proc parentModule*(g: ModuleGraph; fileIdx: FileIndex): FileIndex = diff --git a/compiler/modules.nim b/compiler/modules.nim index 132f3788d..a8a9c4df8 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -11,7 +11,7 @@ import ast, astalgo, magicsys, msgs, options, - idents, lexer, passes, syntaxes, llstream, modulegraphs, rod, + idents, lexer, passes, syntaxes, llstream, modulegraphs, lineinfos, pathutils, tables proc resetSystemArtifacts*(g: ModuleGraph) = @@ -93,8 +93,10 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P discard processModule(graph, result, idGeneratorFromModule(result), s) if result == nil: let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - result = loadModuleSym(graph, fileIdx, filename) - if result == nil: + when false: + # XXX entry point for module loading from the rod file + result = loadModuleSym(graph, fileIdx, filename) + when true: result = newModule(graph, fileIdx) result.flags.incl flags registerModule(graph, result) diff --git a/compiler/nilcheck.nim b/compiler/nilcheck.nim index 23f403589..d0ef45d04 100644 --- a/compiler/nilcheck.nim +++ b/compiler/nilcheck.nim @@ -11,9 +11,9 @@ import ast, renderer, intsets, tables, msgs, options, lineinfos, strformat, iden import sequtils, strutils, std / sets # IMPORTANT: notes not up to date, i'll update this comment again -# +# # notes: -# +# # Env: int => nilability # a = b # nilability a <- nilability b @@ -111,7 +111,7 @@ type Symbol = distinct int ## the index of an expression in the pre-indexed sequence of those - ExprIndex = distinct int16 + ExprIndex = distinct int16 ## the set index SetIndex = distinct int @@ -131,7 +131,7 @@ type ## the context for the checker: an instance for each procedure NilCheckerContext = ref object # abstractTime: AbstractTime - # partitions: Partitions + # partitions: Partitions # symbolGraphs: Table[Symbol, ] symbolIndices: Table[Symbol, ExprIndex] ## index for each symbol expressions: SeqOfDistinct[ExprIndex, PNode] ## a sequence of pre-indexed expressions @@ -360,7 +360,7 @@ func `$`(a: Symbol): string = $(a.int) template isConstBracket(n: PNode): bool = - n.kind == nkBracketExpr and n[1].kind in nkLiterals + n.kind == nkBracketExpr and n[1].kind in nkLiterals proc index(ctx: NilCheckerContext, n: PNode): ExprIndex = # echo "n ", n, " ", n.kind @@ -373,7 +373,7 @@ proc index(ctx: NilCheckerContext, n: PNode): ExprIndex = #echo n.kind # internalError(ctx.config, n.info, "expected " & $a & " " & $n & " to have a index") return noExprIndex - # + # #ctx.symbolIndices[symbol(n)] @@ -384,7 +384,7 @@ proc aliasSet(ctx: NilCheckerContext, map: NilMap, index: ExprIndex): IntSet = result = map.sets[map.setIndices[index]] - + proc store(map: NilMap, ctx: NilCheckerContext, index: ExprIndex, value: Nilability, kind: TransitionKind, info: TLineInfo, node: PNode = nil) = if index == noExprIndex: return @@ -414,7 +414,7 @@ proc moveOut(ctx: NilCheckerContext, map: NilMap, target: PNode) = var targetSet = map.sets[targetSetIndex] if targetSet.len > 1: var other: ExprIndex - + for element in targetSet: if element.ExprIndex != targetIndex: other = element.ExprIndex @@ -440,7 +440,7 @@ proc move(ctx: NilCheckerContext, map: NilMap, target: PNode, assigned: PNode) = #echo "move ", target, " ", assigned var targetIndex = ctx.index(target) var assignedIndex: ExprIndex - var targetSetIndex = map.setIndices[targetIndex] + var targetSetIndex = map.setIndices[targetIndex] var assignedSetIndex: SetIndex if assigned.kind == nkSym: assignedIndex = ctx.index(assigned) @@ -497,12 +497,12 @@ proc checkCall(n, ctx, map): Check = result.map = map for i, child in n: discard check(child, ctx, map) - + if i > 0: # var args make a new map with MaybeNil for our node # as it might have been mutated # TODO similar for normal refs and fields: find dependent exprs: brackets - + if child.kind == nkHiddenAddr and not child.typ.isNil and child.typ.kind == tyVar and child.typ[0].kind == tyRef: if not isNew: result.map = newNilMap(map) @@ -526,7 +526,7 @@ proc checkCall(n, ctx, map): Check = isNew = true moveOutDependants(ctx, result.map, child) storeDependants(ctx, result.map, child, MaybeNil) - + if n[0].kind == nkSym and n[0].sym.magic == mNew: # new hidden deref? var value = if n[1].kind == nkHiddenDeref: n[1][0] else: n[1] @@ -552,7 +552,7 @@ template event(b: History): string = of TSafe: "it is safe here as it returns false for isNil" of TPotentialAlias: "it might be changed directly or through an alias" of TDependant: "it might be changed because its base might be changed" - + proc derefWarning(n, ctx, map; kind: Nilability) = ## a warning for potentially unsafe dereference if n.info in ctx.warningLocations: @@ -587,14 +587,14 @@ proc handleNilability(check: Check; n, ctx, map) = else: when defined(nilDebugInfo): message(ctx.config, n.info, hintUser, "can deref " & $n) - + proc checkDeref(n, ctx, map): Check = ## check dereference: deref n should be ok only if n is Safe result = check(n[0], ctx, map) - + handleNilability(result, n[0], ctx, map) - + proc checkRefExpr(n, ctx; check: Check): Check = ## check ref expressions: TODO not sure when this happens result = check @@ -625,7 +625,7 @@ proc checkBracketExpr(n, ctx, map): Check = result = check(n[1], ctx, result.map) result = checkRefExpr(n, ctx, result) # echo n, " ", result.nilability - + template union(l: Nilability, r: Nilability): Nilability = ## unify two states @@ -654,7 +654,7 @@ proc findCommonParent(l: NilMap, r: NilMap): NilMap = result = l.parent while not result.isNil: var rparent = r.parent - while not rparent.isNil: + while not rparent.isNil: if result == rparent: return result rparent = rparent.parent @@ -666,17 +666,17 @@ proc union(ctx: NilCheckerContext, l: NilMap, r: NilMap): NilMap = ## what if they are from different parts of the same tree ## e.g. ## a -> b -> c - ## -> b1 + ## -> b1 ## common then? - ## + ## if l.isNil: return r elif r.isNil: return l - + let common = findCommonParent(l, r) result = newNilMap(common, ctx.expressions.len.int) - + for index, value in l: let h = history(r, index) let info = if h.len > 0: h[^1].info else: TLineInfo(line: 0) # assert h.len > 0 @@ -715,11 +715,11 @@ proc checkAsgn(target: PNode, assigned: PNode; ctx, map): Check = result = check(assigned, ctx, map) else: result = Check(nilability: typeNilability(target.typ), map: map) - + # we need to visit and check those, but we don't use the result for now # is it possible to somehow have another event happen here? discard check(target, ctx, map) - + if result.map.isNil: result.map = map if target.kind in {nkSym, nkDotExpr} or isConstBracket(target): @@ -738,8 +738,8 @@ proc checkAsgn(target: PNode, assigned: PNode; ctx, map): Check = if symbol(elementNode) in ctx.symbolIndices: var elementIndex = ctx.index(elementNode) result.map.store(ctx, elementIndex, value, TAssign, target.info, elementNode) - - + + proc checkReturn(n, ctx, map): Check = ## check return # return n same as result = n; return ? @@ -750,9 +750,9 @@ proc checkReturn(n, ctx, map): Check = proc checkIf(n, ctx, map): Check = ## check branches based on condition var mapIf: NilMap = map - + # first visit the condition - + # the structure is not If(Elif(Elif, Else), Else) # it is # If(Elif, Elif, Else) @@ -765,7 +765,7 @@ proc checkIf(n, ctx, map): Check = var afterLayer: NilMap # the result nilability for expressions var nilability = Safe - + for branch in n.sons: var branchConditionLayer = newNilMap(layerHistory) var branchLayer: NilMap @@ -779,7 +779,7 @@ proc checkIf(n, ctx, map): Check = else: branchLayer = layerHistory code = branch - + let branchCheck = checkBranch(code, ctx, branchLayer) # handles nil afterLayer -> returns branchCheck.map afterLayer = ctx.union(afterLayer, branchCheck.map) @@ -796,7 +796,7 @@ proc checkIf(n, ctx, map): Check = result.map = ctx.union(layerHistory, afterLayer) result.nilability = Safe # no expr? else: - # similar to else: because otherwise we are jumping out of + # similar to else: because otherwise we are jumping out of # the branch, so no union with the mapIf (we dont continue if the condition was true) # here it also doesn't matter for the parent branch what happened in the branch, e.g. assigning to nil # as if we continue there, we haven't entered the branch probably @@ -820,7 +820,7 @@ proc checkFor(n, ctx, map): Check = # echo namedMapDebugInfo(ctx, map) var check2 = check(n.sons[2], ctx, m) var map2 = check2.map - + result.map = ctx.union(map0, m) result.map = ctx.union(result.map, map2) result.nilability = Safe @@ -848,11 +848,11 @@ proc checkWhile(n, ctx, map): Check = var map1 = m.copyMap() var check2 = check(n.sons[1], ctx, m) var map2 = check2.map - + result.map = ctx.union(map0, map1) result.map = ctx.union(result.map, map2) result.nilability = Safe - + proc checkInfix(n, ctx, map): Check = ## check infix operators in condition ## a and b : map is based on a; next b @@ -882,7 +882,7 @@ proc checkInfix(n, ctx, map): Check = result.map = checkCondition(n[2], ctx, map, false, false) elif $n[1] == "false": result.map = checkCondition(n[2], ctx, map, true, false) - + if result.map.isNil: result.map = map else: @@ -906,24 +906,24 @@ proc infix(ctx: NilCheckerContext, l: PNode, r: PNode, magic: TMagic): PNode = else: "" var cache = newIdentCache() - var op = newSym(skVar, cache.getIdent(name), nextId ctx.idgen, nil, r.info) + var op = newSym(skVar, cache.getIdent(name), nextSymId ctx.idgen, nil, r.info) op.magic = magic result = nkInfix.newTree( newSymNode(op, r.info), l, r) - result.typ = newType(tyBool, nextId ctx.idgen, nil) + result.typ = newType(tyBool, nextTypeId ctx.idgen, nil) proc prefixNot(ctx: NilCheckerContext, node: PNode): PNode = var cache = newIdentCache() - var op = newSym(skVar, cache.getIdent("not"), nextId ctx.idgen, nil, node.info) + var op = newSym(skVar, cache.getIdent("not"), nextSymId ctx.idgen, nil, node.info) op.magic = mNot result = nkPrefix.newTree( newSymNode(op, node.info), node) - result.typ = newType(tyBool, nextId ctx.idgen, nil) + result.typ = newType(tyBool, nextTypeId ctx.idgen, nil) proc infixEq(ctx: NilCheckerContext, l: PNode, r: PNode): PNode = infix(ctx, l, r, mEqRef) @@ -1016,7 +1016,7 @@ proc checkTry(n, ctx, map): Check = let tryCheck = check(n[0], ctx, currentMap) newMap = ctx.union(currentMap, tryCheck.map) canRaise = n[0].canRaise - + var afterTryMap = newMap for a, branch in n: if a > 0: @@ -1026,7 +1026,7 @@ proc checkTry(n, ctx, map): Check = let childCheck = check(branch[0], ctx, newMap) newMap = ctx.union(newMap, childCheck.map) hasFinally = true - of nkExceptBranch: + of nkExceptBranch: if canRaise: let childCheck = check(branch[^1], ctx, newMap) newMap = ctx.union(newMap, childCheck.map) @@ -1069,7 +1069,7 @@ proc reverse(kind: TransitionKind): TransitionKind = of TNil: TSafe of TSafe: TNil of TPotentialAlias: TPotentialAlias - else: + else: kind # raise newException(ValueError, "expected TNil or TSafe") @@ -1079,41 +1079,41 @@ proc reverseDirect(map: NilMap): NilMap = # because conditions should've stored their changes there # b: Safe (not b.isNil) # b: Parent Parent - # b: Nil (b.isNil) + # b: Nil (b.isNil) # layer block # [ Parent ] [ Parent ] - # if -> if state + # if -> if state # layer -> reverse # older older0 new # older new # [ b Nil ] [ Parent ] # elif # [ b Nil, c Nil] [ Parent ] - # + # - # if b.isNil: + # if b.isNil: # # [ b Safe] # c = A() # Safe - # elif not b.isNil: + # elif not b.isNil: # # [ b Safe ] + [b Nil] MaybeNil Unreachable # # Unreachable defer can't deref b, it is unreachable # discard # else: - # b + # b - -# if + +# if # if: we just pass the map with a new layer for its block # elif: we just pass the original map but with a new layer is the reverse of the previous popped layer (?) - # elif: + # elif: # else: we just pass the original map but with a new layer which is initialized as the reverse of the # top layer of else # else: - # + # # [ b MaybeNil ] [b Parent] [b Parent] [b Safe] [b Nil] [] # Safe # c == 1 @@ -1181,7 +1181,7 @@ proc checkResult(n, ctx, map) = of Unreachable: message(ctx.config, n.info, warnStrictNotNil, "return value is unreachable") of Safe, Parent: - discard + discard proc checkBranch(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = result = check(n, ctx, map) @@ -1191,7 +1191,7 @@ proc checkBranch(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = proc check(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = assert not map.isNil - + # echo "check n ", n, " ", n.kind # echo "map ", namedMapDebugInfo(ctx, map) case n.kind: @@ -1218,7 +1218,7 @@ proc check(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = if n.kind in {nkObjConstr, nkTupleConstr}: # TODO deeper nested elements? # A(field: B()) # - # field: Safe -> + # field: Safe -> var elements: seq[(PNode, Nilability)] for i, child in n: result = check(child, ctx, result.map) @@ -1230,7 +1230,7 @@ proc check(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = else: for child in n: result = check(child, ctx, result.map) - + of nkDotExpr: result = checkDotExpr(n, ctx, map) of nkDerefExpr, nkHiddenDeref: @@ -1261,7 +1261,7 @@ proc check(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, nkTypeOfExpr: - + discard "don't follow this : same as varpartitions" result = Check(nilability: Nil, map: map) else: @@ -1275,17 +1275,17 @@ proc check(n: PNode, ctx: NilCheckerContext, map: NilMap): Check = result = Check(nilability: Nil, map: elementCheck.map) - - + + proc typeNilability(typ: PType): Nilability = assert not typ.isNil # echo "typeNilability ", $typ.flags, " ", $typ.kind result = if tfNotNil in typ.flags: Safe elif typ.kind in {tyRef, tyCString, tyPtr, tyPointer}: - # + # # tyVar ? tyVarargs ? tySink ? tyLent ? - # TODO spec? tests? + # TODO spec? tests? MaybeNil else: Safe @@ -1354,7 +1354,7 @@ proc checkNil*(s: PSym; body: PNode; conf: ConfigRef, idgen: IdGenerator) = var context = NilCheckerContext(config: conf, idgen: idgen) context.preVisit(s, body, conf) var map = newNilMap(nil, context.symbolIndices.len) - + for i, child in s.typ.n.sons: if i > 0: if child.kind != nkSym: @@ -1362,7 +1362,7 @@ proc checkNil*(s: PSym; body: PNode; conf: ConfigRef, idgen: IdGenerator) = map.store(context, context.index(child), typeNilability(child.typ), TArg, child.info, child) map.store(context, resultExprIndex, if not s.typ[0].isNil and s.typ[0].kind == tyRef: Nil else: Safe, TResult, s.ast.info) - + # echo "checking ", s.name.s, " ", filename let res = check(body, context, map) @@ -1374,7 +1374,7 @@ proc checkNil*(s: PSym; body: PNode; conf: ConfigRef, idgen: IdGenerator) = res.map.store(context, resultExprIndex, Safe, TAssign, s.ast.info) # TODO check for nilability result - # (ANotNil, BNotNil) : + # (ANotNil, BNotNil) : # do we check on asgn nilability at all? if not s.typ[0].isNil and s.typ[0].kind == tyRef and tfNotNil in s.typ[0].flags: diff --git a/compiler/options.nim b/compiler/options.nim index 1de6d531a..4a013f8fc 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -722,6 +722,10 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, result = subdir / RelativeFile f.string.splitPath.tail #echo "completeGeneratedFilePath(", f, ") = ", result +proc toRodFile*(conf: ConfigRef; f: AbsoluteFile): AbsoluteFile = + result = changeFileExt(completeGeneratedFilePath(conf, + withPackageName(conf, f)), RodExt) + proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile = for it in conf.searchPaths: if suppressStdlib and it.string.startsWith(conf.libpath.string): diff --git a/compiler/passes.nim b/compiler/passes.nim index f266d2a6b..997a10cd8 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -13,7 +13,7 @@ import options, ast, llstream, msgs, idents, - syntaxes, modulegraphs, reorder, rod, + syntaxes, modulegraphs, reorder, lineinfos, pathutils type @@ -119,88 +119,65 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; s: PLLStream fileIdx = module.fileIdx prepareConfigNotes(graph, module) - if module.id < 0: - # new module caching mechanism: - for i in 0..') - -proc encodeType(g: ModuleGraph, t: PType, result: var string) = - if t == nil: - # nil nodes have to be stored too: - result.add("[]") - return - # we need no surrounding [] here because the type is in a line of its own - if t.kind == tyForward: internalError(g.config, "encodeType: tyForward") - # for the new rodfile viewer we use a preceding [ so that the data section - # can easily be disambiguated: - result.add('[') - encodeVInt(ord(t.kind), result) - result.add('+') - encodeVInt(t.uniqueId, result) - if t.id != t.uniqueId: - result.add('+') - encodeVInt(t.id, result) - if t.n != nil: - encodeNode(g, unknownLineInfo, t.n, result) - if t.flags != {}: - result.add('$') - encodeVInt(cast[int32](t.flags), result) - if t.callConv != low(t.callConv): - result.add('?') - encodeVInt(ord(t.callConv), result) - if t.owner != nil: - result.add('*') - encodeVInt(t.owner.id, result) - pushSym(w, t.owner) - if t.sym != nil: - result.add('&') - encodeVInt(t.sym.id, result) - pushSym(w, t.sym) - if t.size != - 1: - result.add('/') - encodeVBiggestInt(t.size, result) - if t.align != 2: - result.add('=') - encodeVInt(t.align, result) - if t.lockLevel.ord != UnspecifiedLockLevel.ord: - result.add('\14') - encodeVInt(t.lockLevel.int16, result) - if t.paddingAtEnd != 0: - result.add('\15') - encodeVInt(t.paddingAtEnd, result) - for a in t.attachedOps: - result.add('\16') - if a == nil: - encodeVInt(-1, result) - else: - encodeVInt(a.id, result) - pushSym(w, a) - for i, s in items(t.methods): - result.add('\19') - encodeVInt(i, result) - result.add('\20') - encodeVInt(s.id, result) - pushSym(w, s) - encodeLoc(g, t.loc, result) - if t.typeInst != nil: - result.add('\21') - encodeVInt(t.typeInst.uniqueId, result) - pushType(w, t.typeInst) - for i in 0.. 100_000: - doAssert false, "loop never ends!" - if w.sstack.len > 0: - let s = w.sstack.pop() - when false: - echo "popped ", s.name.s, " ", s.id - storeSym(g, s) - elif w.tstack.len > 0: - let t = w.tstack.pop() - storeType(g, t) - when false: - echo "popped type ", typeToString(t), " ", t.uniqueId - else: - break - inc i - -proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) = - if g.config.symbolFiles == disabledSf: return - var buf = newStringOfCap(160) - encodeNode(g, module.info, n, buf) - db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)", - abs(module.id), module.offset, buf) - inc module.offset - transitiveClosure(g) - -proc recordStmt*(g: ModuleGraph; module: PSym; n: PNode) = - storeNode(g, module, n) - -proc storeFilename(g: ModuleGraph; fullpath: AbsoluteFile; fileIdx: FileIndex) = - let id = db.getValue(sql"select id from filenames where fullpath = ?", fullpath.string) - if id.len == 0: - let fullhash = hashFileCached(g.config, fileIdx, fullpath) - db.exec(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)", - int(fileIdx), fullpath.string, fullhash) - -proc storeRemaining*(g: ModuleGraph; module: PSym) = - if g.config.symbolFiles == disabledSf: return - var stillForwarded: seq[PSym] = @[] - for s in w.forwardedSyms: - if sfForward notin s.flags: - storeSym(g, s) - else: - stillForwarded.add s - swap w.forwardedSyms, stillForwarded - transitiveClosure(g) - var nimid = 0 - for x in items(g.config.m.fileInfos): - storeFilename(g, x.fullPath, FileIndex(nimid)) - inc nimid - -# ---------------- decoder ----------------------------------- - -type - BlobReader = object - s: string - pos: int - -using - b: var BlobReader - g: ModuleGraph - -proc loadSym(g; id: int, info: TLineInfo): PSym -proc loadType(g; id: int, info: TLineInfo): PType - -proc decodeLineInfo(g; b; info: var TLineInfo) = - if b.s[b.pos] == '?': - inc(b.pos) - if b.s[b.pos] == ',': info.col = -1'i16 - else: info.col = int16(decodeVInt(b.s, b.pos)) - if b.s[b.pos] == ',': - inc(b.pos) - if b.s[b.pos] == ',': info.line = 0'u16 - else: info.line = uint16(decodeVInt(b.s, b.pos)) - if b.s[b.pos] == ',': - inc(b.pos) - #info.fileIndex = fromDbFileId(g.incr, g.config, decodeVInt(b.s, b.pos)) - info.fileIndex = FileIndex decodeVInt(b.s, b.pos) - -proc skipNode(b) = - # ')' itself cannot be part of a string literal so that this is correct. - assert b.s[b.pos] == '(' - var par = 0 - var pos = b.pos+1 - while true: - case b.s[pos] - of ')': - if par == 0: break - dec par - of '(': inc par - else: discard - inc pos - b.pos = pos+1 # skip ')' - -proc decodeNodeLazyBody(g; b; fInfo: TLineInfo, - belongsTo: PSym): PNode = - result = nil - if b.s[b.pos] == '(': - inc(b.pos) - if b.s[b.pos] == ')': - inc(b.pos) - return # nil node - result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo) - decodeLineInfo(g, b, result.info) - if b.s[b.pos] == '$': - inc(b.pos) - result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos))) - if b.s[b.pos] == '^': - inc(b.pos) - var id = decodeVInt(b.s, b.pos) - result.typ = loadType(g, id, result.info) - case result.kind - of nkCharLit..nkUInt64Lit: - if b.s[b.pos] == '!': - inc(b.pos) - result.intVal = decodeVBiggestInt(b.s, b.pos) - of nkFloatLit..nkFloat64Lit: - if b.s[b.pos] == '!': - inc(b.pos) - var fl = decodeStr(b.s, b.pos) - result.floatVal = parseFloat(fl) - of nkStrLit..nkTripleStrLit: - if b.s[b.pos] == '!': - inc(b.pos) - result.strVal = decodeStr(b.s, b.pos) - else: - result.strVal = "" - of nkIdent: - if b.s[b.pos] == '!': - inc(b.pos) - var fl = decodeStr(b.s, b.pos) - result.ident = g.cache.getIdent(fl) - else: - internalError(g.config, result.info, "decodeNode: nkIdent") - of nkSym: - if b.s[b.pos] == '!': - inc(b.pos) - var id = decodeVInt(b.s, b.pos) - result.sym = loadSym(g, id, result.info) - else: - internalError(g.config, result.info, "decodeNode: nkSym") - else: - var i = 0 - while b.s[b.pos] != ')': - when false: - if belongsTo != nil and i == bodyPos: - addSonNilAllowed(result, nil) - belongsTo.offset = b.pos - skipNode(b) - else: - discard - addSonNilAllowed(result, decodeNodeLazyBody(g, b, result.info, nil)) - inc i - if b.s[b.pos] == ')': inc(b.pos) - else: internalError(g.config, result.info, "decodeNode: ')' missing") - else: - internalError(g.config, fInfo, "decodeNode: '(' missing " & $b.pos) - -proc decodeNode(g; b; fInfo: TLineInfo): PNode = - result = decodeNodeLazyBody(g, b, fInfo, nil) - -proc decodeLoc(g; b; loc: var TLoc, info: TLineInfo) = - if b.s[b.pos] == '<': - inc(b.pos) - if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}: - loc.k = TLocKind(decodeVInt(b.s, b.pos)) - else: - loc.k = low(loc.k) - if b.s[b.pos] == '*': - inc(b.pos) - loc.storage = TStorageLoc(decodeVInt(b.s, b.pos)) - else: - loc.storage = low(loc.storage) - if b.s[b.pos] == '$': - inc(b.pos) - loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos))) - else: - loc.flags = {} - if b.s[b.pos] == '^': - inc(b.pos) - loc.lode = decodeNode(g, b, info) - # rrGetType(b, decodeVInt(b.s, b.pos), info) - else: - loc.lode = nil - if b.s[b.pos] == '!': - inc(b.pos) - loc.r = rope(decodeStr(b.s, b.pos)) - else: - loc.r = nil - if b.s[b.pos] == '>': inc(b.pos) - else: internalError(g.config, info, "decodeLoc " & b.s[b.pos]) - -proc loadBlob(g; query: SqlQuery; id: int): BlobReader = - let blob = db.getValue(query, id) - if blob.len == 0: - internalError(g.config, "symbolfiles: cannot find ID " & $ id) - result = BlobReader(pos: 0) - shallowCopy(result.s, blob) - # ensure we can read without index checks: - result.s.add '\0' - -proc loadType(g; id: int; info: TLineInfo): PType = - result = g.incr.r.types.getOrDefault(id) - if result != nil: return result - var b = loadBlob(g, sql"select data from types where nimid = ?", id) - - if b.s[b.pos] == '[': - inc(b.pos) - if b.s[b.pos] == ']': - inc(b.pos) - return # nil type - new(result) - result.kind = TTypeKind(decodeVInt(b.s, b.pos)) - if b.s[b.pos] == '+': - inc(b.pos) - result.uniqueId = decodeVInt(b.s, b.pos) - setId(result.uniqueId) - #if debugIds: registerID(result) - else: - internalError(g.config, info, "decodeType: no id") - if b.s[b.pos] == '+': - inc(b.pos) - result.id = decodeVInt(b.s, b.pos) - else: - result.id = result.uniqueId - # here this also avoids endless recursion for recursive type - g.incr.r.types.add(result.uniqueId, result) - if b.s[b.pos] == '(': result.n = decodeNode(g, b, unknownLineInfo) - if b.s[b.pos] == '$': - inc(b.pos) - result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos))) - if b.s[b.pos] == '?': - inc(b.pos) - result.callConv = TCallingConvention(decodeVInt(b.s, b.pos)) - if b.s[b.pos] == '*': - inc(b.pos) - result.owner = loadSym(g, decodeVInt(b.s, b.pos), info) - if b.s[b.pos] == '&': - inc(b.pos) - result.sym = loadSym(g, decodeVInt(b.s, b.pos), info) - if b.s[b.pos] == '/': - inc(b.pos) - result.size = decodeVInt(b.s, b.pos) - else: - result.size = -1 - if b.s[b.pos] == '=': - inc(b.pos) - result.align = decodeVInt(b.s, b.pos).int16 - else: - result.align = 2 - - if b.s[b.pos] == '\14': - inc(b.pos) - result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel - else: - result.lockLevel = UnspecifiedLockLevel - - if b.s[b.pos] == '\15': - inc(b.pos) - result.paddingAtEnd = decodeVInt(b.s, b.pos).int16 - - for a in low(result.attachedOps)..high(result.attachedOps): - if b.s[b.pos] == '\16': - inc(b.pos) - let id = decodeVInt(b.s, b.pos) - if id >= 0: - result.attachedOps[a] = loadSym(g, id, info) - - while b.s[b.pos] == '\19': - inc(b.pos) - let x = decodeVInt(b.s, b.pos) - doAssert b.s[b.pos] == '\20' - inc(b.pos) - let y = loadSym(g, decodeVInt(b.s, b.pos), info) - result.methods.add((x, y)) - decodeLoc(g, b, result.loc, info) - if b.s[b.pos] == '\21': - inc(b.pos) - let d = decodeVInt(b.s, b.pos) - result.typeInst = loadType(g, d, info) - while b.s[b.pos] == '^': - inc(b.pos) - if b.s[b.pos] == '(': - inc(b.pos) - if b.s[b.pos] == ')': inc(b.pos) - else: internalError(g.config, info, "decodeType ^(" & b.s[b.pos]) - rawAddSon(result, nil) - else: - let d = decodeVInt(b.s, b.pos) - result.sons.add loadType(g, d, info) - -proc decodeLib(g; b; info: TLineInfo): PLib = - result = nil - if b.s[b.pos] == '|': - new(result) - inc(b.pos) - result.kind = TLibKind(decodeVInt(b.s, b.pos)) - if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 1") - inc(b.pos) - result.name = rope(decodeStr(b.s, b.pos)) - if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 2") - inc(b.pos) - result.path = decodeNode(g, b, info) - -proc decodeInstantiations(g; b; info: TLineInfo; - s: var seq[PInstantiation]) = - while b.s[b.pos] == '\15': - inc(b.pos) - var ii: PInstantiation - new ii - ii.sym = loadSym(g, decodeVInt(b.s, b.pos), info) - ii.concreteTypes = @[] - while b.s[b.pos] == '\17': - inc(b.pos) - ii.concreteTypes.add loadType(g, decodeVInt(b.s, b.pos), info) - if b.s[b.pos] == '\20': - inc(b.pos) - ii.compilesId = decodeVInt(b.s, b.pos) - s.add ii - -proc loadSymFromBlob(g; b; info: TLineInfo): PSym = - if b.s[b.pos] == '{': - inc(b.pos) - if b.s[b.pos] == '}': - inc(b.pos) - return # nil sym - var k = TSymKind(decodeVInt(b.s, b.pos)) - var id: int - if b.s[b.pos] == '+': - inc(b.pos) - id = decodeVInt(b.s, b.pos) - setId(id) - else: - internalError(g.config, info, "decodeSym: no id") - var ident: PIdent - if b.s[b.pos] == '&': - inc(b.pos) - ident = g.cache.getIdent(decodeStr(b.s, b.pos)) - else: - internalError(g.config, info, "decodeSym: no ident") - #echo "decoding: {", ident.s - result = PSym(id: id, kind: k, name: ident) - # read the rest of the symbol description: - g.incr.r.syms.add(result.id, result) - if b.s[b.pos] == '^': - inc(b.pos) - result.typ = loadType(g, decodeVInt(b.s, b.pos), info) - decodeLineInfo(g, b, result.info) - if b.s[b.pos] == '*': - inc(b.pos) - result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info) - if b.s[b.pos] == '$': - inc(b.pos) - result.flags = cast[TSymFlags](decodeVBiggestInt(b.s, b.pos)) - if b.s[b.pos] == '@': - inc(b.pos) - result.magic = TMagic(decodeVInt(b.s, b.pos)) - if b.s[b.pos] == '!': - inc(b.pos) - result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos))) - if b.s[b.pos] == '%': - inc(b.pos) - result.position = decodeVInt(b.s, b.pos) - if b.s[b.pos] == '`': - inc(b.pos) - result.offset = decodeVInt(b.s, b.pos) - else: - result.offset = -1 - decodeLoc(g, b, result.loc, result.info) - result.annex = decodeLib(g, b, info) - if b.s[b.pos] == '#': - inc(b.pos) - result.constraint = decodeNode(g, b, unknownLineInfo) - case result.kind - of skType, skGenericParam: - while b.s[b.pos] == '\14': - inc(b.pos) - result.typeInstCache.add loadType(g, decodeVInt(b.s, b.pos), result.info) - of routineKinds: - decodeInstantiations(g, b, result.info, result.procInstCache) - if b.s[b.pos] == '\16': - inc(b.pos) - result.gcUnsafetyReason = loadSym(g, decodeVInt(b.s, b.pos), result.info) - if b.s[b.pos] == '\24': - inc b.pos - result.transformedBody = decodeNode(g, b, result.info) - #result.transformedBody = nil - of skModule, skPackage: - decodeInstantiations(g, b, result.info, result.usedGenerics) - of skLet, skVar, skField, skForVar: - if b.s[b.pos] == '\18': - inc(b.pos) - result.guard = loadSym(g, decodeVInt(b.s, b.pos), result.info) - if b.s[b.pos] == '\19': - inc(b.pos) - result.bitsize = decodeVInt(b.s, b.pos).int16 - else: discard - - if b.s[b.pos] == '(': - #if result.kind in routineKinds: - # result.ast = nil - #else: - result.ast = decodeNode(g, b, result.info) - if sfCompilerProc in result.flags: - registerCompilerProc(g, result) - #echo "loading ", result.name.s - -proc loadSym(g; id: int; info: TLineInfo): PSym = - result = g.incr.r.syms.getOrDefault(id) - if result != nil: return result - var b = loadBlob(g, sql"select data from syms where nimid = ?", id) - result = loadSymFromBlob(g, b, info) - doAssert id == result.id, "symbol ID is not consistent!" - -proc registerModule*(g; module: PSym) = - g.incr.r.syms.add(abs module.id, module) - -proc loadModuleSymTab(g; module: PSym) = - ## goal: fill module.tab - g.incr.r.syms.add(module.id, module) - for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)): - let id = parseInt(row[0]) - var s = g.incr.r.syms.getOrDefault(id) - if s == nil: - var b = BlobReader(pos: 0) - shallowCopy(b.s, row[1]) - # ensure we can read without index checks: - b.s.add '\0' - s = loadSymFromBlob(g, b, module.info) - assert s != nil - if s.kind != skField: - strTableAdd(module.tab, s) - if sfSystemModule in module.flags: - g.systemModule = module - -proc replay(g: ModuleGraph; module: PSym; n: PNode) = - # XXX check if we need to replay nkStaticStmt here. - case n.kind - #of nkStaticStmt: - #evalStaticStmt(module, g, n[0], module) - #of nkVarSection, nkLetSection: - # nkVarSections are already covered by the vmgen which produces nkStaticStmt - of nkMethodDef: - methodDef(g, n[namePos].sym, fromCache=true) - of nkCommentStmt: - # pragmas are complex and can be user-overriden via templates. So - # instead of using the original ``nkPragma`` nodes, we rely on the - # fact that pragmas.nim was patched to produce specialized recorded - # statements for us in the form of ``nkCommentStmt`` with (key, value) - # pairs. Ordinary nkCommentStmt nodes never have children so this is - # not ambiguous. - # Fortunately only a tiny subset of the available pragmas need to - # be replayed here. This is always a subset of ``pragmas.stmtPragmas``. - if n.len >= 2: - internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit - case n[0].strVal - of "hint": message(g.config, n.info, hintUser, n[1].strVal) - of "warning": message(g.config, n.info, warnUser, n[1].strVal) - of "error": localError(g.config, n.info, errUser, n[1].strVal) - of "compile": - internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit - let cname = AbsoluteFile n[1].strVal - var cf = Cfile(nimname: splitFile(cname).name, cname: cname, - obj: AbsoluteFile n[2].strVal, - flags: {CfileFlag.External}, - customArgs: n[3].strVal) - extccomp.addExternalFileToCompile(g.config, cf) - of "link": - extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal) - of "passl": - extccomp.addLinkOption(g.config, n[1].strVal) - of "passc": - extccomp.addCompileOption(g.config, n[1].strVal) - of "localpassc": - extccomp.addLocalCompileOption(g.config, n[1].strVal, toFullPathConsiderDirty(g.config, module.info.fileIndex)) - of "cppdefine": - options.cppDefine(g.config, n[1].strVal) - of "inc": - let destKey = n[1].strVal - let by = n[2].intVal - let v = getOrDefault(g.cacheCounters, destKey) - g.cacheCounters[destKey] = v+by - of "put": - let destKey = n[1].strVal - let key = n[2].strVal - let val = n[3] - if not contains(g.cacheTables, destKey): - g.cacheTables[destKey] = initBTree[string, PNode]() - if not contains(g.cacheTables[destKey], key): - g.cacheTables[destKey].add(key, val) - else: - internalError(g.config, n.info, "key already exists: " & key) - of "incl": - let destKey = n[1].strVal - let val = n[2] - if not contains(g.cacheSeqs, destKey): - g.cacheSeqs[destKey] = newTree(nkStmtList, val) - else: - block search: - for existing in g.cacheSeqs[destKey]: - if exprStructuralEquivalent(existing, val, strictSymEquality=true): - break search - g.cacheSeqs[destKey].add val - of "add": - let destKey = n[1].strVal - let val = n[2] - if not contains(g.cacheSeqs, destKey): - g.cacheSeqs[destKey] = newTree(nkStmtList, val) - else: - g.cacheSeqs[destKey].add val - else: - internalAssert g.config, false - of nkImportStmt: - for x in n: - internalAssert g.config, x.kind == nkSym - let modpath = AbsoluteFile toFullPath(g.config, x.sym.info) - let imported = g.importModuleCallback(g, module, fileInfoIdx(g.config, modpath)) - internalAssert g.config, imported.id < 0 - of nkStmtList, nkStmtListExpr: - for x in n: replay(g, module, x) - of nkExportStmt: - for x in n: - doAssert x.kind == nkSym - strTableAdd(module.tab, x.sym) - else: discard "nothing to do for this node" - -proc loadNode*(g: ModuleGraph; module: PSym): PNode = - loadModuleSymTab(g, module) - result = newNodeI(nkStmtList, module.info) - for row in db.rows(sql"select data from toplevelstmts where module = ? order by position asc", - abs module.id): - var b = BlobReader(pos: 0) - # ensure we can read without index checks: - b.s = row[0] & '\0' - result.add decodeNode(g, b, module.info) - db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId) - replay(g, module, result) - -proc setupModuleCache*(g: ModuleGraph) = - # historical note: there used to be a `rodfiles` dir with special tests - # for incremental compilation via symbol files. This was likely replaced by ic. - if g.config.symbolFiles == disabledSf: return - g.recordStmt = recordStmt - let dbfile = getNimcacheDir(g.config) / RelativeFile"rodfiles.db" - if g.config.symbolFiles == writeOnlySf: - removeFile(dbfile) - createDir getNimcacheDir(g.config) - let ec = encodeConfig(g) - if not fileExists(dbfile): - db = open(connection=string dbfile, user="nim", password="", - database="nim") - createDb(db) - db.exec(sql"insert into config(config) values (?)", ec) - else: - db = open(connection=string dbfile, user="nim", password="", - database="nim") - let oldConfig = db.getValue(sql"select config from config") - g.incr.configChanged = oldConfig != ec - # ensure the filename IDs stay consistent: - for row in db.rows(sql"select fullpath, nimid from filenames order by nimid"): - let id = fileInfoIdx(g.config, AbsoluteFile row[0]) - doAssert id.int == parseInt(row[1]) - db.exec(sql"update config set config = ?", ec) - db.exec(sql"pragma journal_mode=off") - # This MUST be turned off, otherwise it's way too slow even for testing purposes: - db.exec(sql"pragma SYNCHRONOUS=off") - db.exec(sql"pragma LOCKING_MODE=exclusive") - let lastId = db.getValue(sql"select max(idgen) from controlblock") - if lastId.len > 0: - idgen.setId(parseInt lastId) diff --git a/compiler/sem.nim b/compiler/sem.nim index e95f3799c..d5ae5a21d 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -16,7 +16,7 @@ import procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch, intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity, - lowerings, plugins/active, rod, lineinfos, strtabs, int128, + lowerings, plugins/active, lineinfos, strtabs, int128, isolation_check, typeallowed from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward @@ -124,8 +124,8 @@ proc commonType*(c: PContext; x, y: PType): PType = # turn any concrete typedesc into the abstract typedesc type if a.len == 0: result = a else: - result = newType(tyTypeDesc, nextId(c.idgen), a.owner) - rawAddSon(result, newType(tyNone, nextId(c.idgen), a.owner)) + result = newType(tyTypeDesc, nextTypeId(c.idgen), a.owner) + rawAddSon(result, newType(tyNone, nextTypeId(c.idgen), a.owner)) elif b.kind in {tyArray, tySet, tySequence} and a.kind == b.kind: # check for seq[empty] vs. seq[int] @@ -137,7 +137,7 @@ proc commonType*(c: PContext; x, y: PType): PType = let aEmpty = isEmptyContainer(a[i]) let bEmpty = isEmptyContainer(b[i]) if aEmpty != bEmpty: - if nt.isNil: nt = copyType(a, nextId(c.idgen), a.owner) + if nt.isNil: nt = copyType(a, nextTypeId(c.idgen), a.owner) nt[i] = if aEmpty: b[i] else: a[i] if not nt.isNil: result = nt #elif b[idx].kind == tyEmpty: return x @@ -176,7 +176,7 @@ proc commonType*(c: PContext; x, y: PType): PType = # ill-formed AST, no need for additional tyRef/tyPtr if k != tyNone and x.kind != tyGenericInst: let r = result - result = newType(k, nextId(c.idgen), r.owner) + result = newType(k, nextTypeId(c.idgen), r.owner) result.addSonSkipIntLit(r, c.idgen) proc endsInNoReturn(n: PNode): bool = @@ -193,7 +193,7 @@ proc commonType*(c: PContext; x: PType, y: PNode): PType = commonType(c, x, y.typ) proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = - result = newSym(kind, considerQuotedIdent(c, n), nextId c.idgen, getCurrOwner(c), n.info) + result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info) when defined(nimsuggest): suggestDecl(c, n, result) @@ -216,7 +216,7 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = # template; we must fix it here: see #909 result.owner = getCurrOwner(c) else: - result = newSym(kind, considerQuotedIdent(c, n), nextId c.idgen, getCurrOwner(c), n.info) + result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info) #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule: # incl(result.flags, sfGlobal) when defined(nimsuggest): @@ -255,7 +255,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, proc symFromType(c: PContext; t: PType, info: TLineInfo): PSym = if t.sym != nil: return t.sym - result = newSym(skType, getIdent(c.cache, "AnonType"), nextId c.idgen, t.owner, info) + result = newSym(skType, getIdent(c.cache, "AnonType"), nextSymId c.idgen, t.owner, info) result.flags.incl sfAnon result.typ = t @@ -616,7 +616,7 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = else: result = newNodeI(nkEmpty, n.info) #if c.config.cmd == cmdIdeTools: findSuggest(c, n) - rod.storeNode(c.graph, c.module, result) + storeRodNode(c, result) proc reportUnusedModules(c: PContext) = for i in 0..high(c.unusedImports): @@ -638,7 +638,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = result.add(c.module.ast) popOwner(c) popProcCon(c) - storeRemaining(c.graph, c.module) + saveRodFile(c) const semPass* = makePass(myOpen, myProcess, myClose, isFrontend = true) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index a6660e14c..da38a6fc2 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -11,7 +11,9 @@ import intsets, options, ast, astalgo, msgs, idents, renderer, - magicsys, vmdef, modulegraphs, lineinfos, sets + magicsys, vmdef, modulegraphs, lineinfos, sets, pathutils + +import ic / to_packed_ast type TOptionEntry* = object # entries to put on a stack for pragma parsing @@ -77,9 +79,9 @@ type case mode*: ImportMode of importAll: discard of importSet: - imported*: IntSet + imported*: IntSet # of PIdent.id of importExcept: - exceptSet*: IntSet + exceptSet*: IntSet # of PIdent.id PContext* = ref TContext TContext* = object of TPassContext # a context represents the module @@ -140,6 +142,7 @@ type selfName*: PIdent cache*: IdentCache graph*: ModuleGraph + encoder*: PackedEncoder signatures*: TStrTable recursiveDep*: string suggestionsMade*: bool @@ -264,6 +267,16 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext = initStrTable(result.signatures) result.typesWithOps = @[] result.features = graph.config.features + if graph.config.symbolFiles != disabledSf: + initEncoder result.encoder, module, graph.config + +proc addIncludeFileDep*(c: PContext; f: FileIndex) = + if c.config.symbolFiles != disabledSf: + addIncludeFileDep(c.encoder, f) + +proc addImportFileDep*(c: PContext; f: FileIndex) = + if c.config.symbolFiles != disabledSf: + addImportFileDep(c.encoder, f) proc inclSym(sq: var seq[PSym], s: PSym) = for i in 0.. ord(high(TSymKind)): internalError(c.config, c.debug[pc], "request to create symbol of invalid kind") - var sym = newSym(k.TSymKind, getIdent(c.cache, name), nextId c.idgen, c.module.owner, c.debug[pc]) + var sym = newSym(k.TSymKind, getIdent(c.cache, name), nextSymId c.idgen, c.module.owner, c.debug[pc]) incl(sym.flags, sfGenSym) regs[ra].node = newSymNode(sym) regs[ra].node.flags.incl nfIsRef @@ -2257,7 +2257,7 @@ const evalMacroLimit = 1000 proc errorNode(idgen: IdGenerator; owner: PSym, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) - result.typ = newType(tyError, nextId idgen, owner) + result.typ = newType(tyError, nextTypeId idgen, owner) result.typ.flags.incl tfCheckedForDestructor proc evalMacroCall*(module: PSym; idgen: IdGenerator; g: ModuleGraph; templInstCounter: ref int; diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 8083ae179..f8765c4dc 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -26,7 +26,7 @@ proc opSlurp*(file: string, info: TLineInfo, module: PSym; conf: ConfigRef): str proc atomicTypeX(cache: IdentCache; name: string; m: TMagic; t: PType; info: TLineInfo; idgen: IdGenerator): PNode = - let sym = newSym(skType, getIdent(cache, name), nextId(idgen), t.owner, info) + let sym = newSym(skType, getIdent(cache, name), nextSymId(idgen), t.owner, info) sym.magic = m sym.typ = t result = newSymNode(sym) @@ -47,7 +47,7 @@ proc mapTypeToBracketX(cache: IdentCache; name: string; m: TMagic; t: PType; inf for i in 0..= result.len: setLen(result.sons, pos + 1) let fieldNode = newNode(nkExprColonExpr) - fieldNode.add newSymNode(newSym(skField, ident, nextId(idgen), nil, unknownLineInfo)) + fieldNode.add newSymNode(newSym(skField, ident, nextSymId(idgen), nil, unknownLineInfo)) fieldNode.add loadAny(p, field.typ, tab, cache, conf, idgen) result[pos] = fieldNode if p.kind == jsonObjectEnd: next(p) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 9fe934f89..15727c9f7 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -455,9 +455,9 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto = # from a proc, `this` being the first argument. proc replaceSyms(n: NimNode): NimNode = - if n.kind == nnkSym: + if n.kind == nnkSym: result = newIdentNode($n) - else: + else: result = n for i in 0..