refactorings to prepare the compiler for IC (#15935)

* added ic specific Nim code; WIP
* make the symbol import mechanism lazy; WIP
* ensure that modules can be imported multiple times
* ambiguity checking
* handle converters and TR macros properly
* make 'enum' test category green again
* special logic for semi-pure enums
* makes nimsuggest tests green again
* fixes nimdata
* makes nimpy green again
* makes more important packages work

compiler/ast.nim

@@ -799,6 +799,7 @@ type
     libHeader, libDynamic
 
   TLib* = object              # also misused for headers!
+                              # keep in sync with PackedLib
     kind*: TLibKind
     generated*: bool          # needed for the backends:
     isOverriden*: bool
@@ -823,7 +824,7 @@ type
   PScope* = ref TScope
 
   PLib* = ref TLib
-  TSym* {.acyclic.} = object of TIdObj
+  TSym* {.acyclic.} = object of TIdObj # Keep in sync with PackedSym
     # proc and type instantiations are cached in the generic symbol
     case kind*: TSymKind
     of skType, skGenericParam:
@@ -904,6 +905,7 @@ type
                               # types are identical iff they have the
                               # same id; there may be multiple copies of a type
                               # in memory!
+                              # Keep in sync with PackedType
     kind*: TTypeKind          # kind of type
     callConv*: TCallingConvention # for procs
     flags*: TTypeFlags        # flags of the type

compiler/ic/bitabs.nim

@@ -0,0 +1,119 @@
+## A BiTable is a table that can be seen as an optimized pair
+## of (Table[LitId, Val], Table[Val, LitId]).
+
+import hashes
+
+type
+  LitId* = distinct uint32
+
+  BiTable*[T] = object
+    vals: seq[T] # indexed by LitId
+    keys: seq[LitId]  # indexed by hash(val)
+
+proc nextTry(h, maxHash: Hash): Hash {.inline.} =
+  result = (h + 1) and maxHash
+
+template maxHash(t): untyped = high(t.keys)
+template isFilled(x: LitId): bool = x.uint32 > 0'u32
+
+proc `$`*(x: LitId): string {.borrow.}
+proc `<`*(x, y: LitId): bool {.borrow.}
+proc `<=`*(x, y: LitId): bool {.borrow.}
+proc `==`*(x, y: LitId): bool {.borrow.}
+proc hash*(x: LitId): Hash {.borrow.}
+
+
+proc len*[T](t: BiTable[T]): int = t.vals.len
+
+proc mustRehash(length, counter: int): bool {.inline.} =
+  assert(length > counter)
+  result = (length * 2 < counter * 3) or (length - counter < 4)
+
+const
+  idStart = 256 # Ids do not start with 0 but with this value. The IR needs it.
+
+template idToIdx(x: LitId): int = x.int - idStart
+
+proc enlarge[T](t: var BiTable[T]) =
+  var n: seq[LitId]
+  newSeq(n, len(t.keys) * 2)
+  swap(t.keys, n)
+  for i in 0..high(n):
+    let eh = n[i]
+    if isFilled(eh):
+      var j = hash(t.vals[idToIdx eh]) and maxHash(t)
+      while isFilled(t.keys[j]):
+        j = nextTry(j, maxHash(t))
+      t.keys[j] = move n[i]
+
+proc getKeyId*[T](t: BiTable[T]; v: T): LitId =
+  let origH = hash(v)
+  var h = origH and maxHash(t)
+  if t.keys.len != 0:
+    while true:
+      let litId = t.keys[h]
+      if not isFilled(litId): break
+      if t.vals[idToIdx t.keys[h]] == v: return litId
+      h = nextTry(h, maxHash(t))
+  return LitId(0)
+
+proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
+  let origH = hash(v)
+  var h = origH and maxHash(t)
+  if t.keys.len != 0:
+    while true:
+      let litId = t.keys[h]
+      if not isFilled(litId): break
+      if t.vals[idToIdx t.keys[h]] == v: return litId
+      h = nextTry(h, maxHash(t))
+    # not found, we need to insert it:
+    if mustRehash(t.keys.len, t.vals.len):
+      enlarge(t)
+      # recompute where to insert:
+      h = origH and maxHash(t)
+      while true:
+        let litId = t.keys[h]
+        if not isFilled(litId): break
+        h = nextTry(h, maxHash(t))
+  else:
+    setLen(t.keys, 16)
+    h = origH and maxHash(t)
+
+  result = LitId(t.vals.len + idStart)
+  t.keys[h] = result
+  t.vals.add v
+
+
+proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} =
+  let idx = idToIdx LitId
+  assert idx < t.vals.len
+  result = t.vals[idx]
+
+proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
+  let idx = idToIdx LitId
+  assert idx < t.vals.len
+  result = t.vals[idx]
+
+when isMainModule:
+
+  var t: BiTable[string]
+
+  echo getOrIncl(t, "hello")
+
+  echo getOrIncl(t, "hello")
+  echo getOrIncl(t, "hello3")
+  echo getOrIncl(t, "hello4")
+  echo getOrIncl(t, "helloasfasdfdsa")
+  echo getOrIncl(t, "hello")
+  echo getKeyId(t, "hello")
+  echo getKeyId(t, "none")
+
+  for i in 0 ..< 100_000:
+    discard t.getOrIncl($i & "___" & $i)
+
+  for i in 0 ..< 100_000:
+    assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4
+  echo t.vals.len
+
+  echo t.vals[0]
+  echo t.vals[1004]

compiler/ic/design.rst

@@ -0,0 +1,42 @@
+====================================
+  Incremental Recompilations
+====================================
+
+We split the Nim compiler into a frontend and a backend.
+The frontend produces a set of `.rod` files. Every `.nim` module
+produces its own `.rod` file.
+
+- The IR must be a faithful representation of the AST in memory.
+- The backend can do its own caching but doesn't have to.
+- We know by comparing 'nim check compiler/nim' against 'nim c compiler/nim'
+  that 2/3 of the compiler's runtime is spent in the frontend. Hence we
+  implement IC for the frontend first and only later for the backend. The
+  backend will recompile everything until we implement its own caching
+  mechanisms.
+
+Advantage of the "set of files" vs the previous global database:
+- By construction, we either read from the `.rod` file or from the
+  `.nim` file, there can be no inconsistency. There can also be no
+  partial updates.
+- No dependency to external packages (SQLite). SQLite simply is too
+  slow and the old way of serialization was too slow too. We use a
+  format designed for Nim and expect to base further tools on this
+  file format.
+
+References to external modules must be (moduleId, symId) pairs.
+The symbol IDs are module specific. This way no global ID increment
+mechanism needs to be implemented that we could get wrong. ModuleIds
+are rod-file specific too.
+
+
+Configuration setup changes
+---------------------------
+
+For a MVP these are not detected. Later the configuration will be
+stored in every `.rod` file.
+
+
+Global state
+------------
+
+Global persistent state will be kept in a project specific `.rod` file.

compiler/ic/from_packed_ast.nim

@@ -0,0 +1,12 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import std / [hashes, tables]
+import bitabs
+import ".." / [ast, lineinfos, options, pathutils]

compiler/ic/packed_ast.nim

@@ -0,0 +1,461 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Packed AST representation, mostly based on a seq of nodes.
+## For IC support. Far future: Rewrite the compiler passes to
+## use this representation directly in all the transformations,
+## it is superior.
+
+import std / [hashes, tables]
+import bitabs
+import ".." / [ast, lineinfos, options, pathutils]
+
+const
+  localNamePos* = 0
+  localExportMarkerPos* = 1
+  localPragmaPos* = 2
+  localTypePos* = 3
+  localValuePos* = 4
+
+  typeNamePos* = 0
+  typeExportMarkerPos* = 1
+  typeGenericParamsPos* = 2
+  typePragmaPos* = 3
+  typeBodyPos* = 4
+
+  routineNamePos* = 0
+  routineExportMarkerPos* = 1
+  routinePatternPos* = 2
+  routineGenericParamsPos* = 3
+  routineParamsPos* = 4
+  routineResultPos* = 5
+  routinePragmasPos* = 6
+  routineBodyPos* = 7
+
+const
+  nkModuleRef = nkNone # pair of (ModuleId, SymId)
+
+type
+  SymId* = distinct int32
+  TypeId* = distinct int32
+  ModuleId* = distinct int32
+  NodePos* = distinct int
+
+  NodeId* = distinct int32
+
+  PackedLineInfo* = object
+    line*: uint16
+    col*: int16
+    file*: LitId
+
+  PackedLib* = object
+    kind*: TLibKind
+    generated*: bool
+    isOverriden*: bool
+    name*: LitId
+    path*: NodeId
+
+  PackedSym* = object
+    kind*: TSymKind
+    name*: LitId
+    typeId*: TypeId
+    flags*: TSymFlags
+    magic*: TMagic
+    info*: PackedLineInfo
+    ast*: NodePos
+    owner*: ItemId
+    guard*: ItemId
+    bitsize*: int
+    alignment*: int # for alignment
+    options*: TOptions
+    position*: int
+    offset*: int
+    externalName*: LitId # instead of TLoc
+    annex*: PackedLib
+    when hasFFI:
+      cname*: LitId
+    constraint*: NodeId
+
+  PackedType* = object
+    kind*: TTypeKind
+    nodekind*: TNodeKind
+    flags*: TTypeFlags
+    types*: int32
+    nodes*: int32
+    methods*: int32
+    nodeflags*: TNodeFlags
+    info*: PackedLineInfo
+    sym*: ItemId
+    owner*: ItemId
+    attachedOps*: array[TTypeAttachedOp, ItemId]
+    size*: BiggestInt
+    align*: int16
+    paddingAtEnd*: int16
+    lockLevel*: TLockLevel # lock level as required for deadlock checking
+    # not serialized: loc*: TLoc because it is backend-specific
+    typeInst*: TypeId
+    nonUniqueId*: ItemId
+
+  Node* = object     # 20 bytes
+    kind*: TNodeKind
+    flags*: TNodeFlags
+    operand*: int32  # for kind in {nkSym, nkSymDef}: SymId
+                     # for kind in {nkStrLit, nkIdent, nkNumberLit}: LitId
+                     # for kind in nkInt32Lit: direct value
+                     # for non-atom kinds: the number of nodes (for easy skipping)
+    typeId*: TypeId
+    info*: PackedLineInfo
+
+  ModulePhase* = enum
+    preLookup, lookedUpTopLevelStmts
+
+  Module* = object
+    name*: string
+    file*: AbsoluteFile
+    ast*: PackedTree
+    phase*: ModulePhase
+    iface*: Table[string, seq[SymId]] # 'seq' because of overloading
+
+  Program* = ref object
+    modules*: seq[Module]
+
+  Shared* = ref object # shared between different versions of 'Module'.
+                       # (though there is always exactly one valid
+                       # version of a module)
+    syms*: seq[PackedSym]
+    types*: seq[seq[Node]]
+    strings*: BiTable[string] # we could share these between modules.
+    integers*: BiTable[BiggestInt]
+    floats*: BiTable[BiggestFloat]
+    config*: ConfigRef
+    #thisModule*: ModuleId
+    #program*: Program
+
+  PackedTree* = object ## usually represents a full Nim module
+    nodes*: seq[Node]
+    toPosition*: Table[SymId, NodePos]
+    sh*: Shared
+
+proc `==`*(a, b: SymId): bool {.borrow.}
+proc hash*(a: SymId): Hash {.borrow.}
+
+proc `==`*(a, b: NodePos): bool {.borrow.}
+proc `==`*(a, b: TypeId): bool {.borrow.}
+proc `==`*(a, b: ModuleId): bool {.borrow.}
+
+proc declareSym*(tree: var PackedTree; kind: TSymKind;
+                 name: LitId; info: PackedLineInfo): SymId =
+  result = SymId(tree.sh.syms.len)
+  tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
+
+proc newTreeFrom*(old: PackedTree): PackedTree =
+  result.nodes = @[]
+  result.sh = old.sh
+
+proc litIdFromName*(tree: PackedTree; name: string): LitId =
+  result = tree.sh.strings.getOrIncl(name)
+
+proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: kind, operand: int32 getOrIncl(tree.sh.strings, token), info: info)
+
+proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: kind, operand: 0, info: info)
+
+proc throwAwayLastNode*(tree: var PackedTree) =
+  tree.nodes.setLen(tree.nodes.len-1)
+
+proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: nkIdent, operand: int32(s), info: info)
+
+proc addSym*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
+
+proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: nkInt32Lit, operand: int32(s), info: info)
+
+proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
+  tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
+
+proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
+
+proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
+  # and this is why the IR is superior. We can copy subtrees
+  # via a linear scan.
+  let pos = n.int
+  let L = if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
+  let d = dest.nodes.len
+  dest.nodes.setLen(d + L)
+  for i in 0..<L:
+    dest.nodes[d+i] = tree.nodes[pos+i]
+
+proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
+  result = SymId(dest.sh.syms.len)
+  assert int(s) < tree.sh.syms.len
+  let oldSym = tree.sh.syms[s.int]
+  dest.sh.syms.add oldSym
+
+type
+  PatchPos = distinct int
+
+when false:
+  proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
+    result = PatchPos tree.nodes.len
+    tree.nodes.add Node(kind: kind, operand: 0, info: info)
+
+proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: TypeId; info: PackedLineInfo): PatchPos =
+  result = PatchPos tree.nodes.len
+  tree.nodes.add Node(kind: kind, flags: flags, operand: 0, typeId: typeId, info: info)
+
+proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
+  result = PatchPos dest.nodes.len
+  dest.nodes.add source.nodes[sourcePos.int]
+
+proc patch*(tree: var PackedTree; pos: PatchPos) =
+  let pos = pos.int
+  assert tree.nodes[pos].kind > nkNilLit
+  let distance = int32(tree.nodes.len - pos)
+  tree.nodes[pos].operand = distance
+
+proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
+
+proc `[]`*(tree: PackedTree; i: int): lent Node {.inline.} = tree.nodes[i]
+
+proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
+  if tree.nodes[pos].kind > nkNilLit:
+    assert tree.nodes[pos].operand > 0
+    inc pos, tree.nodes[pos].operand
+  else:
+    inc pos
+
+iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos =
+  var pos = n.int
+  assert tree.nodes[pos].kind > nkNilLit
+  let last = pos + tree.nodes[pos].operand
+  inc pos
+  while pos < last:
+    yield NodePos pos
+    nextChild tree, pos
+
+iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
+  let patchPos = prepare(dest, tree, n)
+  for x in sonsReadonly(tree, n): yield x
+  patch dest, patchPos
+
+iterator isons*(dest: var PackedTree; tree: PackedTree; n: NodePos): (int, NodePos) =
+  var i = 0
+  for ch0 in sons(dest, tree, n):
+    yield (i, ch0)
+    inc i
+
+iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos =
+  var pos = n.int
+  assert tree.nodes[pos].kind > nkNilLit
+  let last = pos + tree.nodes[pos].operand
+  inc pos
+  if pos < last:
+    nextChild tree, pos
+  while pos < last:
+    yield NodePos pos
+    nextChild tree, pos
+
+iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos =
+  var count = 0
+  for child in sonsReadonly(tree, n):
+    inc count
+  var pos = n.int
+  assert tree.nodes[pos].kind > nkNilLit
+  let last = pos + tree.nodes[pos].operand
+  inc pos
+  while pos < last and count > 2:
+    yield NodePos pos
+    dec count
+    nextChild tree, pos
+
+proc parentImpl(tree: PackedTree; n: NodePos): NodePos =
+  # finding the parent of a node is rather easy:
+  var pos = n.int - 1
+  while pos >= 0 and isAtom(tree, pos) or (pos + tree.nodes[pos].operand - 1 < n.int):
+    dec pos
+  assert pos >= 0, "node has no parent"
+  result = NodePos(pos)
+
+template parent*(n: NodePos): NodePos = parentImpl(tree, n)
+
+proc hasXsons*(tree: PackedTree; n: NodePos; x: int): bool =
+  var count = 0
+  if tree.nodes[n.int].kind > nkNilLit:
+    for child in sonsReadonly(tree, n): inc count
+  result = count == x
+
+proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
+  if tree.nodes[n.int].kind > nkNilLit:
+    var count = 0
+    for child in sonsReadonly(tree, n):
+      inc count
+      if count >= x: return true
+  return false
+
+proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
+proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind
+proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = LitId tree.nodes[n.int].operand
+proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info
+
+proc span(tree: PackedTree; pos: int): int {.inline.} =
+  if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
+
+proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) =
+  assert(not isAtom(tree, n.int))
+  let a = n.int+1
+  let b = a + span(tree, a)
+  result = (NodePos a, NodePos b)
+
+proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) =
+  assert(not isAtom(tree, n.int))
+  let a = n.int+1
+  let b = a + span(tree, a)
+  let c = b + span(tree, b)
+  result = (NodePos a, NodePos b, NodePos c)
+
+proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
+  if tree.nodes[n.int].kind > nkNilLit:
+    var count = 0
+    for child in sonsReadonly(tree, n):
+      if count == i: return child
+      inc count
+  assert false, "node has no i-th child"
+
+proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = tree.sh.strings[lit]
+
+template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
+template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
+template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].operand
+
+template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
+
+proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
+
+proc strLit*(tree: PackedTree; n: NodePos): lent string =
+  assert n.kind == nkStrLit
+  result = tree.sh.strings[LitId tree.nodes[n.int].operand]
+
+proc strVal*(tree: PackedTree; n: NodePos): string =
+  assert n.kind == nkStrLit
+  result = tree.sh.strings[LitId tree.nodes[n.int].operand]
+  #result = cookedStrLit(raw)
+
+proc filenameVal*(tree: PackedTree; n: NodePos): string =
+  case n.kind
+  of nkStrLit:
+    result = strVal(tree, n)
+  of nkIdent:
+    result = tree.sh.strings[n.litId]
+  of nkSym:
+    result = tree.sh.strings[tree.sh.syms[int n.symId].name]
+  else:
+    result = ""
+
+proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
+  assert n.kind == nkIdent
+  result = tree.sh.strings[LitId tree.nodes[n.int].operand]
+
+const
+  externIntLit* = {nkCharLit,
+    nkIntLit,
+    nkInt8Lit,
+    nkInt16Lit,
+    nkInt64Lit,
+    nkUIntLit,
+    nkUInt8Lit,
+    nkUInt16Lit,
+    nkUInt32Lit,
+    nkUInt64Lit} # nkInt32Lit is missing by design!
+
+  externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt64Lit}
+  externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
+  directIntLit* = nkInt32Lit
+
+proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
+  let pos = n.int
+  if result.len > 0 and result[^1] notin {' ', '\n'}:
+    result.add ' '
+
+  result.add $tree[pos].kind
+  case tree.nodes[pos].kind
+  of nkNone, nkEmpty, nkNilLit, nkType: discard
+  of nkIdent, nkStrLit..nkTripleStrLit:
+    result.add " "
+    result.add tree.sh.strings[LitId tree.nodes[pos].operand]
+  of nkSym:
+    result.add " "
+    result.add tree.sh.strings[tree.sh.syms[tree.nodes[pos].operand].name]
+  of directIntLit:
+    result.add " "
+    result.addInt tree.nodes[pos].operand
+  of externSIntLit:
+    result.add " "
+    result.addInt tree.sh.integers[LitId tree.nodes[pos].operand]
+  of externUIntLit:
+    result.add " "
+    result.add $cast[uint64](tree.sh.integers[LitId tree.nodes[pos].operand])
+  else:
+    result.add "(\n"
+    for i in 1..(nesting+1)*2: result.add ' '
+    for child in sonsReadonly(tree, n):
+      toString(tree, child, nesting + 1, result)
+    result.add "\n"
+    for i in 1..nesting*2: result.add ' '
+    result.add ")"
+    #for i in 1..nesting*2: result.add ' '
+
+
+proc toString*(tree: PackedTree; n: NodePos): string =
+  result = ""
+  toString(tree, n, 0, result)
+
+proc debug*(tree: PackedTree) =
+  stdout.write toString(tree, NodePos 0)
+
+proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
+  if n.kind == nkIdent:
+    result = n.litId
+  elif n.kind == nkSym:
+    result = tree.sh.syms[int n.symId].name
+  else:
+    result = LitId(0)
+
+template identId*(n: NodePos): LitId = identIdImpl(tree, n)
+
+template copyInto*(dest, n, body) =
+  let patchPos = prepare(dest, tree, n)
+  body
+  patch dest, patchPos
+
+template copyIntoKind*(dest, kind, info, body) =
+  let patchPos = prepare(dest, kind, info)
+  body
+  patch dest, patchPos
+
+proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
+  let litId = tree.sh.strings.getKeyId(pragma)
+  if litId == LitId(0):
+    return false
+  assert n.kind == nkPragma
+  for ch0 in sonsReadonly(tree, n):
+    if ch0.kind == nkExprColonExpr:
+      if ch0.firstSon.identId == litId:
+        return true
+    elif ch0.identId == litId:
+      return true
+
+when false:
+  proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
+    let patchPos = prepare(dest, nkError, n.info)
+    dest.add nkStrLit, msg, n.info
+    copyTree(dest, tree, n)
+    patch dest, patchPos

compiler/ic/to_packed_ast.nim

@@ -0,0 +1,90 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import std / [hashes, tables]
+import packed_ast, bitabs
+import ".." / [ast, idents, lineinfos, options, pathutils, msgs]
+
+type
+  Context = object
+    thisModule: int32
+    lastFile: FileIndex # remember the last lookup entry.
+    lastLit: LitId
+    filenames: Table[FileIndex, LitId]
+
+proc toLitId(x: FileIndex; ir: var PackedTree; c: var Context): LitId =
+  if x == c.lastFile:
+    result = c.lastLit
+  else:
+    result = c.filenames.getOrDefault(x)
+    if result == LitId(0):
+      let p = msgs.toFullPath(ir.sh.config, x)
+      result = getOrIncl(ir.sh.strings, p)
+      c.filenames[x] = result
+    c.lastFile = x
+    c.lastLit = result
+
+proc toPackedInfo(x: TLineInfo; ir: var PackedTree; c: var Context): PackedLineInfo =
+  PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, ir, c))
+
+proc toPackedType(t: PType; ir: var PackedTree; c: var Context): TypeId =
+  result = TypeId(0)
+
+proc toPackedSym(s: PSym; ir: var PackedTree; c: var Context): SymId =
+  result = SymId(0)
+
+proc toPackedSymNode(n: PNode; ir: var PackedTree; c: var Context) =
+  assert n.kind == nkSym
+  let t = toPackedType(n.typ, ir, c)
+
+  if n.sym.itemId.module == c.thisModule:
+    # it is a symbol that belongs to the module we're currently
+    # packing:
+    let sid = toPackedSym(n.sym, ir, c)
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(sid),
+      typeId: t, info: toPackedInfo(n.info, ir, c))
+  else:
+    # store it as an external module reference:
+    #  nkModuleRef
+    discard
+
+
+proc toPackedNode*(n: PNode; ir: var PackedTree; c: var Context) =
+  template toP(x: TLineInfo): PackedLineInfo = toPackedInfo(x, ir, c)
+
+  case n.kind
+  of nkNone, nkEmpty, nkNilLit:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: 0,
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  of nkIdent:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.ident.s),
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  of nkSym:
+    toPackedSymNode(n, ir, c)
+  of directIntLit:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(n.intVal),
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  of externIntLit:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.integers, n.intVal),
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  of nkStrLit..nkTripleStrLit:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.strVal),
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  of nkFloatLit..nkFloat128Lit:
+    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.floats, n.floatVal),
+      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+  else:
+    let patchPos = ir.prepare(n.kind, n.flags, toPackedType(n.typ, ir, c), toP n.info)
+    for i in 0..<n.len:
+      toPackedNode(n[i], ir, c)
+    ir.patch patchPos
+
+proc moduleToIr*(n: PNode; ir: var PackedTree; module: PSym) =
+  var c = Context(thisModule: module.itemId.module)
+  toPackedNode(n, ir, c)

compiler/importer.nim

@@ -20,30 +20,67 @@ proc readExceptSet*(c: PContext, n: PNode): IntSet =
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
-proc importPureEnumField*(c: PContext; s: PSym) =
+proc declarePureEnumField*(c: PContext; s: PSym) =
-  let check = strTableGet(c.importTable.symbols, s.name)
+  # XXX Remove the outer 'if' statement and see what breaks.
-  if check == nil:
+  var amb = false
-    let checkB = strTableGet(c.pureEnumFields, s.name)
+  if someSymFromImportTable(c, s.name, amb) == nil:
-    if checkB == nil:
+    strTableAdd(c.pureEnumFields, s)
-      strTableAdd(c.pureEnumFields, s)
+    when false:
-    else:
+      let checkB = strTableGet(c.pureEnumFields, s.name)
+      if checkB == nil:
+        strTableAdd(c.pureEnumFields, s)
+    when false:
       # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc rawImportSymbol(c: PContext, s, origin: PSym) =
+proc importPureEnumField(c: PContext; s: PSym) =
+  var amb = false
+  if someSymFromImportTable(c, s.name, amb) == nil:
+    strTableAdd(c.pureEnumFields, s)
+    when false:
+      let checkB = strTableGet(c.pureEnumFields, s.name)
+      if checkB == nil:
+        strTableAdd(c.pureEnumFields, s)
+    when false:
+      # mark as ambiguous:
+      incl(c.ambiguousSymbols, checkB.id)
+      incl(c.ambiguousSymbols, s.id)
+
+proc importPureEnumFields(c: PContext; s: PSym; etyp: PType) =
+  assert sfPure in s.flags
+  for j in 0..<etyp.n.len:
+    var e = etyp.n[j].sym
+    if e.kind != skEnumField:
+      internalError(c.config, s.info, "rawImportSymbol")
+      # BUGFIX: because of aliases for enums the symbol may already
+      # have been put into the symbol table
+      # BUGFIX: but only iff they are the same symbols!
+    for check in importedItems(c, e.name):
+      if check.id == e.id:
+        e = nil
+        break
+    if e != nil:
+      importPureEnumField(c, e)
+
+proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
-  var check = strTableGet(c.importTable.symbols, s.name)
+  when false:
-  if check != nil and check.id != s.id:
+    var check = someSymFromImportTable(c, s.name)
-    if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
+    if check != nil and check.id != s.id:
-      # s and check need to be qualified:
+      if s.kind notin OverloadableSyms or check.kind notin OverloadableSyms:
-      incl(c.ambiguousSymbols, s.id)
+        # s and check need to be qualified:
-      incl(c.ambiguousSymbols, check.id)
+        incl(c.ambiguousSymbols, s.id)
+        incl(c.ambiguousSymbols, check.id)
   # thanks to 'export' feature, it could be we import the same symbol from
-  # multiple sources, so we need to call 'StrTableAdd' here:
+  # multiple sources, so we need to call 'strTableAdd' here:
-  strTableAdd(c.importTable.symbols, s)
+  when false:
+    # now lazy. Speeds up the compiler and is a prerequisite for IC.
+    strTableAdd(c.importTable.symbols, s)
+  else:
+    importSet.incl s.id
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
@@ -54,16 +91,13 @@ proc rawImportSymbol(c: PContext, s, origin: PSym) =
           # BUGFIX: because of aliases for enums the symbol may already
           # have been put into the symbol table
           # BUGFIX: but only iff they are the same symbols!
-        var it: TIdentIter
+        for check in importedItems(c, e.name):
-        check = initIdentIter(it, c.importTable.symbols, e.name)
-        while check != nil:
           if check.id == e.id:
             e = nil
             break
-          check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e, origin)
+            rawImportSymbol(c, e, origin, importSet)
           else:
             importPureEnumField(c, e)
   else:
@@ -72,7 +106,7 @@ proc rawImportSymbol(c: PContext, s, origin: PSym) =
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
-proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
+proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
   let ident = lookups.considerQuotedIdent(c, n)
   let s = strTableGet(fromMod.tab, ident)
   if s == nil:
@@ -89,29 +123,79 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
-          rawImportSymbol(c, e, fromMod)
+          rawImportSymbol(c, e, fromMod, importSet)
         e = nextIdentIter(it, fromMod.tab)
     else:
-      rawImportSymbol(c, s, fromMod)
+      rawImportSymbol(c, s, fromMod, importSet)
     suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
+proc addImport(c: PContext; im: sink ImportedModule) =
+  for i in 0..high(c.imports):
+    if c.imports[i].m == im.m:
+      # we have already imported the module: Check which import
+      # is more "powerful":
+      case c.imports[i].mode
+      of importAll: discard "already imported all symbols"
+      of importSet:
+        case im.mode
+        of importAll, importExcept:
+          # XXX: slightly wrong semantics for 'importExcept'...
+          # But we should probably change the spec and disallow this case.
+          c.imports[i] = im
+        of importSet:
+          # merge the import sets:
+          c.imports[i].imported.incl im.imported
+      of importExcept:
+        case im.mode
+        of importAll:
+          c.imports[i] = im
+        of importSet:
+          discard
+        of importExcept:
+          var cut = initIntSet()
+          # only exclude what is consistent between the two sets:
+          for j in im.exceptSet:
+            if j in c.imports[i].exceptSet:
+              cut.incl j
+          c.imports[i].exceptSet = cut
+      return
+  c.imports.add im
+
+template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
+  for it in c.graph.ifaces[fromMod.position].converters:
+    if filter:
+      addConverter(c, it)
+  for it in c.graph.ifaces[fromMod.position].patterns:
+    if filter:
+      addPattern(c, it)
+  for it in c.graph.ifaces[fromMod.position].pureEnums:
+    if filter:
+      importPureEnumFields(c, it, it.typ)
+
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
-  var i: TTabIter
+  c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  var s = initTabIter(i, fromMod.tab)
+  addUnnamedIt(c, fromMod, it.id notin exceptSet)
-  while s != nil:
+
-    if s.kind != skModule:
+  when false:
-      if s.kind != skEnumField:
+    var i: TTabIter
-        if s.kind notin ExportableSymKinds:
+    var s = initTabIter(i, fromMod.tab)
-          internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
+    while s != nil:
-        if exceptSet.isNil or s.name.id notin exceptSet:
+      if s.kind != skModule:
-          rawImportSymbol(c, s, fromMod)
+        if s.kind != skEnumField:
-    s = nextIter(i, fromMod.tab)
+          if s.kind notin ExportableSymKinds:
+            internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
+          if exceptSet.isNil or s.name.id notin exceptSet:
+            rawImportSymbol(c, s, fromMod)
+      s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
-  var exceptSet: IntSet
+  c.addImport ImportedModule(m: fromMod, mode: importAll)
-  importAllSymbolsExcept(c, fromMod, exceptSet)
+  addUnnamedIt(c, fromMod, true)
+  when false:
+    var exceptSet: IntSet
+    importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; importSet: var IntSet) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -121,12 +205,12 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s, fromMod)
+        rawImportSymbol(c, s, fromMod, importSet)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
     for i in 0..n.safeLen-1:
-      importForwarded(c, n[i], exceptSet, fromMod)
+      importForwarded(c, n[i], exceptSet, fromMod, importSet)
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
@@ -224,9 +308,12 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
   if m != nil:
     n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
+
+    var im = ImportedModule(m: m, mode: importSet, imported: initIntSet())
     for i in 1..<n.len:
       if n[i].kind != nkNilLit:
-        importSymbol(c, n[i], m)
+        importSymbol(c, n[i], m, im.imported)
+    c.addImport im
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)

compiler/lookups.nim

@@ -74,12 +74,17 @@ proc closeScope*(c: PContext) =
   ensureNoMissingOrUnusedSymbols(c, c.currentScope)
   rawCloseScope(c)
 
-iterator walkScopes*(scope: PScope): PScope =
+iterator allScopes(scope: PScope): PScope =
   var current = scope
   while current != nil:
     yield current
     current = current.parent
 
+iterator localScopesFrom*(c: PContext; scope: PScope): PScope =
+  for s in allScopes(scope):
+    if s == c.topLevelScope: break
+    yield s
+
 proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
   if s == nil or s.kind != skAlias:
     result = s
@@ -91,7 +96,8 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
               s.name.s & " is deprecated")
 
-proc isShadowScope*(s: PScope): bool {.inline.} = s.parent != nil and s.parent.depthLevel == s.depthLevel
+proc isShadowScope*(s: PScope): bool {.inline.} =
+  s.parent != nil and s.parent.depthLevel == s.depthLevel
 
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
   var scope = c.currentScope
@@ -101,15 +107,92 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym =
     scope = scope.parent
     result = strTableGet(scope.symbols, s)
 
-proc searchInScopes*(c: PContext, s: PIdent): PSym =
+proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; name: PIdent): PSym =
-  for scope in walkScopes(c.currentScope):
+  result = initIdentIter(ti, im.m.tab, name)
-    result = strTableGet(scope.symbols, s)
+  while result != nil:
-    if result != nil: return
+    let b =
+      case im.mode
+      of importAll: true
+      of importSet: result.id in im.imported
+      of importExcept: name.id notin im.exceptSet
+    if b and not containsOrIncl(marked, result.id):
+      return result
+    result = nextIdentIter(ti, im.m.tab)
+
+proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule): PSym =
+  while true:
+    result = nextIdentIter(ti, im.m.tab)
+    if result == nil: return nil
+    case im.mode
+    of importAll:
+      if not containsOrIncl(marked, result.id):
+        return result
+    of importSet:
+      if result.id in im.imported and not containsOrIncl(marked, result.id):
+        return result
+    of importExcept:
+      if result.name.id notin im.exceptSet and not containsOrIncl(marked, result.id):
+        return result
+
+iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent): PSym =
+  var ti: TIdentIter
+  var candidate = initIdentIter(ti, marked, im, name)
+  while candidate != nil:
+    yield candidate
+    candidate = nextIdentIter(ti, marked, im)
+
+iterator importedItems*(c: PContext; name: PIdent): PSym =
+  var marked = initIntSet()
+  for im in c.imports.mitems:
+    for s in symbols(im, marked, name):
+      yield s
+
+proc allPureEnumFields(c: PContext; name: PIdent): seq[PSym] =
+  var ti: TIdentIter
+  result = @[]
+  var res = initIdentIter(ti, c.pureEnumFields, name)
+  while res != nil:
+    result.add res
+    res = nextIdentIter(ti, c.pureEnumFields)
+
+iterator allSyms*(c: PContext): (PSym, int, bool) =
+  # really iterate over all symbols in all the scopes. This is expensive
+  # and only used by suggest.nim.
+  var isLocal = true
+  var scopeN = 0
+  for scope in allScopes(c.currentScope):
+    if scope == c.topLevelScope: isLocal = false
+    dec scopeN
+    for item in scope.symbols:
+      yield (item, scopeN, isLocal)
+
+  dec scopeN
+  isLocal = false
+  for im in c.imports.mitems:
+    for s in im.m.tab.data:
+      if s != nil:
+        yield (s, scopeN, isLocal)
+
+proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
+  var marked = initIntSet()
   result = nil
+  for im in c.imports.mitems:
+    for s in symbols(im, marked, name):
+      if result == nil:
+        result = s
+      else:
+        if s.kind notin OverloadableSyms or result.kind notin OverloadableSyms:
+          ambiguous = true
+
+proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
+  for scope in allScopes(c.currentScope):
+    result = strTableGet(scope.symbols, s)
+    if result != nil: return result
+  result = someSymFromImportTable(c, s, ambiguous)
 
 proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
   var i = 0
-  for scope in walkScopes(c.currentScope):
+  for scope in allScopes(c.currentScope):
     echo "scope ", i
     for h in 0..high(scope.symbols.data):
       if scope.symbols.data[h] != nil:
@@ -117,14 +200,23 @@ proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
     if i == limit: break
     inc i
 
-proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
+proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] =
-  for scope in walkScopes(c.currentScope):
+  result = @[]
+  for scope in allScopes(c.currentScope):
     var ti: TIdentIter
     var candidate = initIdentIter(ti, scope.symbols, s)
     while candidate != nil:
-      if candidate.kind in filter: return candidate
+      if candidate.kind in filter:
+        if result.len == 0:
+          result.add candidate
       candidate = nextIdentIter(ti, scope.symbols)
-  result = nil
+
+  if result.len == 0:
+    var marked = initIntSet()
+    for im in c.imports.mitems:
+      for s in symbols(im, marked, s):
+        if s.kind in filter:
+          result.add s
 
 proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
@@ -138,9 +230,9 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   result = newSym(skError, ident, nextId(c.idgen), getCurrOwner(c), n.info, {})
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
-  # pretend it's imported from some unknown module to prevent cascading errors:
+  # pretend it's from the top level scope to prevent cascading errors:
   if c.config.cmd != cmdInteractive and c.compilesContextId == 0:
-    c.importTable.addSym(result)
+    c.moduleScope.addSym(result)
 
 type
   TOverloadIterMode* = enum
@@ -151,8 +243,9 @@ type
     m*: PSym
     mode*: TOverloadIterMode
     symChoiceIndex*: int
-    scope*: PScope
+    currentScope: PScope
-    inSymChoice: IntSet
+    importIdx: int
+    marked: IntSet
 
 proc getSymRepr*(conf: ConfigRef; s: PSym, getDeclarationPath = true): string =
   case s.kind
@@ -276,15 +369,23 @@ else:
 
 proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) =
   var err = "ambiguous identifier: '" & s.name.s & "'"
-  var ti: TIdentIter
-  var candidate = initIdentIter(ti, c.importTable.symbols, s.name)
   var i = 0
-  while candidate != nil:
+  for candidate in importedItems(c, s.name):
+    if i == 0: err.add " -- use one of the following:\n"
+    else: err.add "\n"
+    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
+    err.add ": " & typeToString(candidate.typ)
+    inc i
+  localError(c.config, info, errGenerated, err)
+
+proc errorUseQualifier(c: PContext; info: TLineInfo; candidates: seq[PSym]) =
+  var err = "ambiguous identifier: '" & candidates[0].name.s & "'"
+  var i = 0
+  for candidate in candidates:
     if i == 0: err.add " -- use one of the following:\n"
     else: err.add "\n"
     err.add "  " & candidate.owner.name.s & "." & candidate.name.s
     err.add ": " & typeToString(candidate.typ)
-    candidate = nextIdentIter(ti, c.importTable.symbols)
     inc i
   localError(c.config, info, errGenerated, err)
 
@@ -299,9 +400,10 @@ proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string) =
 
 proc lookUp*(c: PContext, n: PNode): PSym =
   # Looks up a symbol. Generates an error in case of nil.
+  var amb = false
   case n.kind
   of nkIdent:
-    result = searchInScopes(c, n.ident).skipAlias(n, c.config)
+    result = searchInScopes(c, n.ident, amb).skipAlias(n, c.config)
     if result == nil:
       fixSpelling(n, n.ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, n.ident.s)
@@ -310,7 +412,7 @@ proc lookUp*(c: PContext, n: PNode): PSym =
     result = n.sym
   of nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
-    result = searchInScopes(c, ident).skipAlias(n, c.config)
+    result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
     if result == nil:
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
@@ -318,7 +420,8 @@ proc lookUp*(c: PContext, n: PNode): PSym =
   else:
     internalError(c.config, n.info, "lookUp")
     return
-  if contains(c.ambiguousSymbols, result.id):
+  if amb:
+    #contains(c.ambiguousSymbols, result.id):
     errorUseQualifier(c, n.info, result)
   when false:
     if result.kind == skStub: loadStub(result)
@@ -328,29 +431,40 @@ type
     checkAmbiguity, checkUndeclared, checkModule, checkPureEnumFields
 
 proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
-  const allExceptModule = {low(TSymKind)..high(TSymKind)}-{skModule,skPackage}
+  const allExceptModule = {low(TSymKind)..high(TSymKind)} - {skModule, skPackage}
   case n.kind
   of nkIdent, nkAccQuoted:
+    var amb = false
     var ident = considerQuotedIdent(c, n)
     if checkModule in flags:
-      result = searchInScopes(c, ident).skipAlias(n, c.config)
+      result = searchInScopes(c, ident, amb).skipAlias(n, c.config)
     else:
-      result = searchInScopes(c, ident, allExceptModule).skipAlias(n, c.config)
+      let candidates = searchInScopesFilterBy(c, ident, allExceptModule) #.skipAlias(n, c.config)
-    if result == nil and checkPureEnumFields in flags:
+      if candidates.len > 0:
-      result = strTableGet(c.pureEnumFields, ident)
+        result = candidates[0]
+        amb = candidates.len > 1
+        if amb and checkAmbiguity in flags:
+          errorUseQualifier(c, n.info, candidates)
+    if result == nil:
+      let candidates = allPureEnumFields(c, ident)
+      if candidates.len > 0:
+        result = candidates[0]
+        amb = candidates.len > 1
+        if amb and checkAmbiguity in flags:
+          errorUseQualifier(c, n.info, candidates)
+
     if result == nil and checkUndeclared in flags:
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
-    elif checkAmbiguity in flags and result != nil and result.id in c.ambiguousSymbols:
+    elif checkAmbiguity in flags and result != nil and amb:
       errorUseQualifier(c, n.info, result)
+    c.isAmbiguous = amb
   of nkSym:
     result = n.sym
-    if checkAmbiguity in flags and contains(c.ambiguousSymbols, result.id):
-      errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags * {checkUndeclared}) + {checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
       if n[1].kind == nkIdent:
@@ -379,18 +493,29 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     if result != nil and result.kind == skStub: loadStub(result)
 
 proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
+  o.importIdx = -1
+  o.marked = initIntSet()
   case n.kind
   of nkIdent, nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
-    o.scope = c.currentScope
+    var scope = c.currentScope
     o.mode = oimNoQualifier
     while true:
-      result = initIdentIter(o.it, o.scope.symbols, ident).skipAlias(n, c.config)
+      result = initIdentIter(o.it, scope.symbols, ident).skipAlias(n, c.config)
       if result != nil:
+        o.currentScope = scope
         break
       else:
-        o.scope = o.scope.parent
+        scope = scope.parent
-        if o.scope == nil: break
+        if scope == nil:
+          for i in 0..c.imports.high:
+            result = initIdentIter(o.it, o.marked, c.imports[i], ident).skipAlias(n, c.config)
+            if result != nil:
+              o.currentScope = nil
+              o.importIdx = i
+              return result
+          return nil
+
   of nkSym:
     result = n.sym
     o.mode = oimDone
@@ -422,31 +547,69 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimDone
       return nil
     o.symChoiceIndex = 1
-    o.inSymChoice = initIntSet()
+    o.marked = initIntSet()
-    incl(o.inSymChoice, result.id)
+    incl(o.marked, result.id)
   else: discard
   when false:
     if result != nil and result.kind == skStub: loadStub(result)
 
 proc lastOverloadScope*(o: TOverloadIter): int =
   case o.mode
-  of oimNoQualifier: result = if o.scope.isNil: -1 else: o.scope.depthLevel
+  of oimNoQualifier:
+    result = if o.importIdx >= 0: 0
+             elif o.currentScope.isNil: -1
+             else: o.currentScope.depthLevel
   of oimSelfModule:  result = 1
   of oimOtherModule: result = 0
   else: result = -1
 
+proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym =
+  assert o.currentScope == nil
+  var idx = o.importIdx+1
+  o.importIdx = c.imports.len # assume the other imported modules lack this symbol too
+  while idx < c.imports.len:
+    result = initIdentIter(o.it, o.marked, c.imports[idx], o.it.name).skipAlias(n, c.config)
+    if result != nil:
+      # oh, we were wrong, some other module had the symbol, so remember that:
+      o.importIdx = idx
+      break
+    inc idx
+
+proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym =
+  assert o.currentScope == nil
+  while o.importIdx < c.imports.len:
+    result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+    #while result != nil and result.id in o.marked:
+    #  result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx])
+    if result != nil:
+      #assert result.id notin o.marked
+      return result
+    inc o.importIdx
+
 proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   case o.mode
   of oimDone:
     result = nil
   of oimNoQualifier:
-    if o.scope != nil:
+    if o.currentScope != nil:
-      result = nextIdentIter(o.it, o.scope.symbols).skipAlias(n, c.config)
+      assert o.importIdx < 0
+      result = nextIdentIter(o.it, o.currentScope.symbols).skipAlias(n, c.config)
       while result == nil:
-        o.scope = o.scope.parent
+        o.currentScope = o.currentScope.parent
-        if o.scope == nil: break
+        if o.currentScope != nil:
-        result = initIdentIter(o.it, o.scope.symbols, o.it.name).skipAlias(n, c.config)
+          result = initIdentIter(o.it, o.currentScope.symbols, o.it.name).skipAlias(n, c.config)
-        # BUGFIX: o.it.name <-> n.ident
+          # BUGFIX: o.it.name <-> n.ident
+        else:
+          o.importIdx = 0
+          if c.imports.len > 0:
+            result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+            if result == nil:
+              result = nextOverloadIterImports(o, c, n)
+          break
+    elif o.importIdx < c.imports.len:
+      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      if result == nil:
+        result = nextOverloadIterImports(o, c, n)
     else:
       result = nil
   of oimSelfModule:
@@ -456,26 +619,48 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimSymChoice:
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym
-      incl(o.inSymChoice, result.id)
+      incl(o.marked, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
       # try 'local' symbols too for Koenig's lookup:
       o.mode = oimSymChoiceLocalLookup
-      o.scope = c.currentScope
+      o.currentScope = c.currentScope
-      result = firstIdentExcluding(o.it, o.scope.symbols,
+      result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.marked).skipAlias(n, c.config)
       while result == nil:
-        o.scope = o.scope.parent
+        o.currentScope = o.currentScope.parent
-        if o.scope == nil: break
+        if o.currentScope != nil:
-        result = firstIdentExcluding(o.it, o.scope.symbols,
+          result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
+        else:
+          o.importIdx = 0
+          result = symChoiceExtension(o, c, n)
+          break
+      if result != nil:
+        incl o.marked, result.id
   of oimSymChoiceLocalLookup:
-    result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
+    if o.currentScope != nil:
-    while result == nil:
+      result = nextIdentExcluding(o.it, o.currentScope.symbols, o.marked).skipAlias(n, c.config)
-      o.scope = o.scope.parent
+      while result == nil:
-      if o.scope == nil: break
+        o.currentScope = o.currentScope.parent
-      result = firstIdentExcluding(o.it, o.scope.symbols,
+        if o.currentScope != nil:
-                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+          result = firstIdentExcluding(o.it, o.currentScope.symbols,
+                                      n[0].sym.name, o.marked).skipAlias(n, c.config)
+        else:
+          o.importIdx = 0
+          result = symChoiceExtension(o, c, n)
+          break
+      if result != nil:
+        incl o.marked, result.id
+
+    elif o.importIdx < c.imports.len:
+      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      #assert result.id notin o.marked
+      #while result != nil and result.id in o.marked:
+      #  result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config)
+      if result == nil:
+        inc o.importIdx
+        result = symChoiceExtension(o, c, n)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/main.nim

@@ -31,9 +31,9 @@ proc semanticPasses(g: ModuleGraph) =
 proc writeDepsFile(g: ModuleGraph) =
   let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
   let f = open(fname.string, fmWrite)
-  for m in g.modules:
+  for m in g.ifaces:
-    if m != nil:
+    if m.module != nil:
-      f.writeLine(toFullPath(g.config, m.position.FileIndex))
+      f.writeLine(toFullPath(g.config, m.module.position.FileIndex))
   for k in g.inclToMod.keys:
     if g.getModule(k).isNil:  # don't repeat includes which are also modules
       f.writeLine(toFullPath(g.config, k))

compiler/modulegraphs.nim

@@ -28,11 +28,20 @@
 import ast, intsets, tables, options, lineinfos, hashes, idents,
   incremental, btrees, md5
 
+import ic / packed_ast
+
 type
   SigHash* = distinct MD5Digest
 
+  Iface* = object       ## data we don't want to store directly in the
+                        ## ast.PSym type for s.kind == skModule
+    module*: PSym       ## module this "Iface" belongs to
+    converters*: seq[PSym]
+    patterns*: seq[PSym]
+    pureEnums*: seq[PSym]
+
   ModuleGraph* = ref object
-    modules*: seq[PSym]  ## indexed by int32 fileIdx
+    ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
@@ -171,13 +180,21 @@ proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
   result.magic = m
   result.flags = {sfNeverRaises}
 
+proc registerModule*(g: ModuleGraph; m: PSym) =
+  assert m != nil
+  assert m.kind == skModule
+
+  if m.position >= g.ifaces.len:
+    setLen(g.ifaces, m.position + 1)
+  g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[])
+
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
   result.idgen = IdGenerator(module: -1'i32, item: 0'i32)
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
   result.importDeps = initTable[FileIndex, seq[FileIndex]]()
-  result.modules = @[]
+  result.ifaces = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
   result.config = config
@@ -201,7 +218,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
   g.deps = initIntSet()
-  g.modules = @[]
+  g.ifaces = @[]
   g.importStack = @[]
   g.inclToMod = initTable[FileIndex, FileIndex]()
   g.usageSym = nil
@@ -211,8 +228,8 @@ proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.exposed)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len:
+  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len:
-    result = g.modules[fileIdx.int32]
+    result = g.ifaces[fileIdx.int32].module
 
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
@@ -233,7 +250,7 @@ proc parentModule*(g: ModuleGraph; fileIdx: FileIndex): FileIndex =
   ## returns 'fileIdx' if the file belonging to this index is
   ## directly used as a module or else the module that first
   ## references this include file.
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.modules.len and g.modules[fileIdx.int32] != nil:
+  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len and g.ifaces[fileIdx.int32].module != nil:
     result = fileIdx
   else:
     result = g.inclToMod.getOrDefault(fileIdx)
@@ -257,11 +274,11 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # cleared but D still needs to be remembered as 'dirty'.
   if g.invalidTransitiveClosure:
     g.invalidTransitiveClosure = false
-    transitiveClosure(g.deps, g.modules.len)
+    transitiveClosure(g.deps, g.ifaces.len)
 
   # every module that *depends* on this file is also dirty:
-  for i in 0i32..<g.modules.len.int32:
+  for i in 0i32..<g.ifaces.len.int32:
-    let m = g.modules[i]
+    let m = g.ifaces[i].module
     if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
       incl m.flags, sfDirty
 

compiler/modules.nim

@@ -59,12 +59,14 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
   result.owner = packSym
   result.position = int fileIdx
 
-  if int(fileIdx) >= graph.modules.len:
+  graph.registerModule(result)
-    setLen(graph.modules, int(fileIdx) + 1)
-  graph.modules[result.position] = result
 
   initStrTable(result.tab)
-  strTableAdd(result.tab, result) # a module knows itself
+  when false:
+    strTableAdd(result.tab, result) # a module knows itself
+    # This is now implemented via
+    #   c.moduleScope.addSym(module) # a module knows itself
+    # in sem.nim, around line 527
   strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =

compiler/plugins/locals.nim

@@ -19,8 +19,7 @@ proc semLocals*(c: PContext, n: PNode): PNode =
   tupleType.n = newNodeI(nkRecList, n.info)
   let owner = getCurrOwner(c)
   # for now we skip openarrays ...
-  for scope in walkScopes(c.currentScope):
+  for scope in localScopesFrom(c, c.currentScope):
-    if scope == c.topLevelScope: break
     for it in items(scope.symbols):
       if it.kind in skLocalVars and
           it.typ.skipTypes({tyGenericInst, tyVar}).kind notin

compiler/pragmas.nim

@@ -559,7 +559,9 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
       if c < 0: sub = substr(str, b + 1)
       else: sub = substr(str, b + 1, c - 1)
       if sub != "":
-        var e = searchInScopes(con, getIdent(con.cache, sub))
+        var amb = false
+        var e = searchInScopes(con, getIdent(con.cache, sub), amb)
+        # XXX what to do here if 'amb' is true?
         if e != nil:
           when false:
             if e.kind == skStub: loadStub(e)

compiler/rod.nim

@@ -22,7 +22,7 @@ when not nimIncremental:
 
   template storeRemaining*(g: ModuleGraph; module: PSym) = discard
 
-  template registerModule*(g: ModuleGraph; module: PSym) = discard
+  #template registerModule*(g: ModuleGraph; module: PSym) = discard
 
 else:
   include rodimpl

compiler/sem.nim

@@ -522,8 +522,10 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
 
   pushProcCon(c, module)
   pushOwner(c, c.module)
-  c.importTable = openScope(c)
+
-  c.importTable.addSym(module) # a module knows itself
+  c.moduleScope = openScope(c)
+  c.moduleScope.addSym(module) # a module knows itself
+
   if sfSystemModule in module.flags:
     graph.systemModule = module
   c.topLevelScope = openScope(c)
@@ -557,7 +559,7 @@ proc isEmptyTree(n: PNode): bool =
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
   if c.topStmts == 0 and not isImportSystemStmt(c.graph, n):
     if sfSystemModule notin c.module.flags and not isEmptyTree(n):
-      c.importTable.addSym c.graph.systemModule # import the "System" identifier
+      c.moduleScope.addSym c.graph.systemModule # import the "System" identifier
       importAllSymbols(c, c.graph.systemModule)
       inc c.topStmts
   else:

compiler/semdata.nim

@@ -70,13 +70,25 @@ type
 
   TExprFlags* = set[TExprFlag]
 
+  ImportMode* = enum
+    importAll, importSet, importExcept
+  ImportedModule* = object
+    m*: PSym
+    case mode*: ImportMode
+    of importAll: discard
+    of importSet:
+      imported*: IntSet
+    of importExcept:
+      exceptSet*: IntSet
+
   PContext* = ref TContext
   TContext* = object of TPassContext # a context represents the module
                                      # that is currently being compiled
     enforceVoidContext*: PType
     module*: PSym              # the module sym belonging to the context
     currentScope*: PScope      # current scope
-    importTable*: PScope       # scope for all imported symbols
+    moduleScope*: PScope       # scope for modules
+    imports*: seq[ImportedModule] # scope for all imported symbols
     topLevelScope*: PScope     # scope for all top-level symbols
     p*: PProcCon               # procedure context
     matchedConcept*: ptr TMatchedConcept # the current concept being matched
@@ -85,9 +97,6 @@ type
                                # can access private object fields
     instCounter*: int          # to prevent endless instantiations
     templInstCounter*: ref int # gives every template instantiation a unique id
-
-    ambiguousSymbols*: IntSet  # ids of all ambiguous symbols (cannot
-                               # store this info in the syms themselves!)
     inGenericContext*: int     # > 0 if we are in a generic type
     inStaticContext*: int      # > 0 if we are inside a static: block
     inUnrolledContext*: int    # > 0 if we are unrolling a loop
@@ -134,6 +143,7 @@ type
     signatures*: TStrTable
     recursiveDep*: string
     suggestionsMade*: bool
+    isAmbiguous*: bool # little hack
     features*: set[Feature]
     inTypeContext*: int
     typesWithOps*: seq[(PType, PType)] #\
@@ -238,7 +248,6 @@ proc popOptionEntry*(c: PContext) =
 proc newContext*(graph: ModuleGraph; module: PSym): PContext =
   new(result)
   result.enforceVoidContext = PType(kind: tyTyped)
-  result.ambiguousSymbols = initIntSet()
   result.optionStack = @[newOptionEntry(graph.config)]
   result.libs = @[]
   result.module = module
@@ -263,9 +272,14 @@ proc inclSym(sq: var seq[PSym], s: PSym) =
 
 proc addConverter*(c: PContext, conv: PSym) =
   inclSym(c.converters, conv)
+  inclSym(c.graph.ifaces[c.module.position].converters, conv)
+
+proc addPureEnum*(c: PContext, e: PSym) =
+  inclSym(c.graph.ifaces[c.module.position].pureEnums, e)
 
 proc addPattern*(c: PContext, p: PSym) =
   inclSym(c.patterns, p)
+  inclSym(c.graph.ifaces[c.module.position].patterns, p)
 
 proc newLib*(kind: TLibKind): PLib =
   new(result)

compiler/semexprs.nim

@@ -495,7 +495,8 @@ proc semOpAux(c: PContext, n: PNode) =
 proc overloadedCallOpr(c: PContext, n: PNode): PNode =
   # quick check if there is *any* () operator overloaded:
   var par = getIdent(c.cache, "()")
-  if searchInScopes(c, par) == nil:
+  var amb = false
+  if searchInScopes(c, par, amb) == nil:
     result = nil
   else:
     result = newNodeI(nkCall, n.info)
@@ -653,7 +654,8 @@ proc hasUnresolvedArgs(c: PContext, n: PNode): bool =
     return isUnresolvedSym(n.sym)
   of nkIdent, nkAccQuoted:
     let ident = considerQuotedIdent(c, n)
-    let sym = searchInScopes(c, ident)
+    var amb = false
+    let sym = searchInScopes(c, ident, amb)
     if sym != nil:
       return isUnresolvedSym(sym)
     else:
@@ -1880,10 +1882,12 @@ proc semDefined(c: PContext, n: PNode): PNode =
 proc lookUpForDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
   case n.kind
   of nkIdent, nkAccQuoted:
+    var amb = false
+    let ident = considerQuotedIdent(c, n)
     result = if onlyCurrentScope:
-               localSearchInScope(c, considerQuotedIdent(c, n))
+               localSearchInScope(c, ident)
              else:
-               searchInScopes(c, considerQuotedIdent(c, n))
+               searchInScopes(c, ident, amb)
   of nkDotExpr:
     result = nil
     if onlyCurrentScope: return
@@ -2531,6 +2535,11 @@ proc semExportExcept(c: PContext, n: PNode): PNode =
   markUsed(c, n.info, exported)
 
 proc semExport(c: PContext, n: PNode): PNode =
+  proc specialSyms(c: PContext; s: PSym) {.inline.} =
+    if s.kind == skConverter: addConverter(c, s)
+    elif s.kind == skType and s.typ != nil and s.typ.kind == tyEnum and sfPure in s.flags:
+      addPureEnum(c, s)
+
   result = newNodeI(nkExportStmt, n.info)
   for i in 0..<n.len:
     let a = n[i]
@@ -2547,6 +2556,7 @@ proc semExport(c: PContext, n: PNode): PNode =
         if it.kind in ExportableSymKinds+{skModule}:
           strTableAdd(c.module.tab, it)
           result.add newSymNode(it, a.info)
+          specialSyms(c, it)
         it = nextIter(ti, s.tab)
       markUsed(c, n.info, s)
     else:
@@ -2558,6 +2568,16 @@ proc semExport(c: PContext, n: PNode): PNode =
           result.add(newSymNode(s, a.info))
           strTableAdd(c.module.tab, s)
           markUsed(c, n.info, s)
+          specialSyms(c, s)
+          if s.kind == skType and sfPure notin s.flags:
+            var etyp = s.typ
+            if etyp.kind in {tyBool, tyEnum}:
+              for j in 0..<etyp.n.len:
+                var e = etyp.n[j].sym
+                if e.kind != skEnumField:
+                  internalError(c.config, s.info, "rawImportSymbol")
+                strTableAdd(c.module.tab, e)
+
         s = nextOverloadIter(o, c, a)
 
 proc semTupleConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
@@ -2722,6 +2742,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     #when defined(nimsuggest):
     #  if gIdeCmd == ideCon and c.config.m.trackPos == n.info: suggestExprNoCheck(c, n)
     let mode = if nfDotField in n.flags: {} else: {checkUndeclared}
+    c.isAmbiguous = false
     var s = qualifiedLookUp(c, n[0], mode)
     if s != nil:
       #if c.config.cmd == cmdNimfix and n[0].kind == nkDotExpr:
@@ -2731,7 +2752,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
         result = semDirectOp(c, n, flags)
       of skType:
         # XXX think about this more (``set`` procs)
-        let ambig = contains(c.ambiguousSymbols, s.id)
+        let ambig = c.isAmbiguous
         if not (n[0].kind in {nkClosedSymChoice, nkOpenSymChoice, nkIdent} and ambig) and n.len == 2:
           result = semConv(c, n)
         elif ambig and n.len == 1:

compiler/semgnrc.nim

@@ -115,11 +115,12 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
             ctx: var GenericCtx): PNode =
   result = n
   let ident = considerQuotedIdent(c, n)
-  var s = searchInScopes(c, ident).skipAlias(n, c.config)
+  var amb = false
+  var s = searchInScopes(c, ident, amb).skipAlias(n, c.config)
   if s == nil:
     s = strTableGet(c.pureEnumFields, ident)
-    if s != nil and contains(c.ambiguousSymbols, s.id):
+    #if s != nil and contains(c.ambiguousSymbols, s.id):
-      s = nil
+    #  s = nil
   if s == nil:
     if ident.id notin ctx.toMixin and withinMixin notin flags:
       errorUndeclaredIdentifier(c, n.info, ident.s)
@@ -152,8 +153,9 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
     result = n
     let n = n[1]
     let ident = considerQuotedIdent(c, n)
-    var s = searchInScopes(c, ident, routineKinds).skipAlias(n, c.config)
+    var candidates = searchInScopesFilterBy(c, ident, routineKinds) # .skipAlias(n, c.config)
-    if s != nil:
+    if candidates.len > 0:
+      let s = candidates[0] # XXX take into account the other candidates!
       isMacro = s.kind in {skTemplate, skMacro}
       if withinBind in flags or s.id in ctx.toBind:
         result = newDot(result, symChoice(c, n, s, scClosed))

compiler/semstmts.nim

@@ -298,8 +298,7 @@ proc fitRemoveHiddenConv(c: PContext, typ: PType, n: PNode): PNode =
     changeType(c, result, typ, check=false)
 
 proc findShadowedVar(c: PContext, v: PSym): PSym =
-  for scope in walkScopes(c.currentScope.parent):
+  for scope in localScopesFrom(c, c.currentScope.parent):
-    if scope == c.topLevelScope: break
     let shadowed = strTableGet(scope.symbols, v.name)
     if shadowed != nil and shadowed.kind in skLocalVars:
       return shadowed
@@ -451,7 +450,9 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode =
       n.kind == nkConstSection and w in constPragmas:
       return nil
 
-    let sym = searchInScopes(c, ident)
+    var amb = false
+    let sym = searchInScopes(c, ident, amb)
+    # XXX what if amb is true?
     if sym == nil or sfCustomPragma in sym.flags: return nil
       # skip if not in scope; skip `template myAttr() {.pragma.}`
     let lhs = b[0]
@@ -1476,7 +1477,8 @@ proc semProcAnnotation(c: PContext, prc: PNode;
       if strTableGet(c.userPragmas, ident) != nil:
         continue # User defined pragma
       else:
-        let sym = searchInScopes(c, ident)
+        var amb = false
+        let sym = searchInScopes(c, ident, amb)
         if sym != nil and sfCustomPragma in sym.flags:
           continue # User custom pragma
 

compiler/semtypes.nim

@@ -146,10 +146,12 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     onDef(e.info, e)
     if sfGenSym notin e.flags:
       if not isPure: addDecl(c, e)
-      else: importPureEnumField(c, e)
+      else: declarePureEnumField(c, e)
     if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil):
       wrongRedefinition(c, e.info, e.name.s, conflict.info)
     inc(counter)
+  if isPure and sfExported in result.sym.flags:
+    addPureEnum(c, result.sym)
   if tfNotNil in e.typ.flags and not hasNull:
     result.flags.incl tfRequiresInit
 
@@ -1593,7 +1595,9 @@ proc applyTypeSectionPragmas(c: PContext; pragmas, operand: PNode): PNode =
       if strTableGet(c.userPragmas, ident) != nil:
         discard "User-defined pragma"
       else:
-        let sym = searchInScopes(c, ident)
+        var amb = false
+        let sym = searchInScopes(c, ident, amb)
+        # XXX: What to do here if amb is true?
         if sym != nil and sfCustomPragma in sym.flags:
           discard "Custom user pragma"
         else:

compiler/suggest.nim

@@ -271,17 +271,12 @@ proc getQuality(s: PSym): range[0..100] =
   return 100
 
 template wholeSymTab(cond, section: untyped) {.dirty.} =
-  var isLocal = true
+  for (item, scopeN, isLocal) in allSyms(c):
-  var scopeN = 0
+    let it = item
-  for scope in walkScopes(c.currentScope):
+    var pm: PrefixMatch
-    if scope == c.topLevelScope: isLocal = false
+    if cond:
-    dec scopeN
+      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it),
-    for item in scope.symbols:
+                                pm, c.inTypeContext > 0, scopeN))
-      let it = item
-      var pm: PrefixMatch
-      if cond:
-        outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it),
-                                 pm, c.inTypeContext > 0, scopeN))
 
 proc suggestSymList(c: PContext, list, f: PNode; info: TLineInfo, outputs: var Suggestions) =
   for i in 0..<list.len:
@@ -348,17 +343,11 @@ proc suggestOperations(c: PContext, n, f: PNode, typ: PType, outputs: var Sugges
 
 proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) =
   # do not produce too many symbols:
-  var isLocal = true
+  for (it, scopeN, isLocal) in allSyms(c):
-  var scopeN = 0
+    var pm: PrefixMatch
-  for scope in walkScopes(c.currentScope):
+    if filterSym(it, f, pm):
-    if scope == c.topLevelScope: isLocal = false
+      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, n.info, 0, pm,
-    dec scopeN
+                                c.inTypeContext > 0, scopeN))
-    for it in items(scope.symbols):
-      var pm: PrefixMatch
-      if filterSym(it, f, pm):
-        outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, n.info, 0, pm,
-                                 c.inTypeContext > 0, scopeN))
-    #if scope == c.topLevelScope and f.isNil: break
 
 proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) =
   # special code that deals with ``myObj.``. `n` is NOT the nkDotExpr-node, but
@@ -656,17 +645,12 @@ proc suggestSentinel*(c: PContext) =
   inc(c.compilesContextId)
   var outputs: Suggestions = @[]
   # suggest everything:
-  var isLocal = true
+  for (it, scopeN, isLocal) in allSyms(c):
-  var scopeN = 0
+    var pm: PrefixMatch
-  for scope in walkScopes(c.currentScope):
+    if filterSymNoOpr(it, nil, pm):
-    if scope == c.topLevelScope: isLocal = false
+      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug,
-    dec scopeN
+          newLineInfo(c.config.m.trackPos.fileIndex, 0, -1), 0,
-    for it in items(scope.symbols):
+          PrefixMatch.None, false, scopeN))
-      var pm: PrefixMatch
-      if filterSymNoOpr(it, nil, pm):
-        outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug,
-            newLineInfo(c.config.m.trackPos.fileIndex, 0, -1), 0,
-            PrefixMatch.None, false, scopeN))
 
   dec(c.compilesContextId)
   produceOutput(outputs, c.config)

lib/system.nim

@@ -627,7 +627,7 @@ proc newSeq*[T](s: var seq[T], len: Natural) {.magic: "NewSeq", noSideEffect.}
   ## the sequence instead of adding them. Example:
   ##
   ## .. code-block:: Nim
-  ##   var inputStrings : seq[string]
+  ##   var inputStrings: seq[string]
   ##   newSeq(inputStrings, 3)
   ##   assert len(inputStrings) == 3
   ##   inputStrings[0] = "The fourth"

testament/important_packages.nim

@@ -76,7 +76,9 @@ pkg2 "nigui", "nim c -o:niguii -r src/nigui.nim"
 pkg2 "NimData", "nim c -o:nimdataa src/nimdata.nim"
 pkg2 "nimes", "nim c src/nimes.nim"
 pkg2 "nimfp", "nim c -o:nfp -r src/fp.nim"
-pkg2 "nimgame2", "nim c nimgame2/nimgame.nim"
+when false:
+  pkg2 "nimgame2", "nim c nimgame2/nimgame.nim"
+  # XXX Doesn't work with deprecated 'randomize', will create a PR.
 pkg2 "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
 pkg2 "nimlsp"
 pkg2 "nimly", "nim c -r tests/test_readme_example.nim"

tests/modules/mforwarded_pure_enum.nim

@@ -0,0 +1,3 @@
+
+import mforwarded_pure_enum2
+export mforwarded_pure_enum2.PureEnum

tests/modules/mforwarded_pure_enum2.nim

@@ -0,0 +1,4 @@
+
+type
+  PureEnum* {.pure.} = enum
+    x, y, z

tests/modules/tfowarded_pure_enum.nim

@@ -0,0 +1,7 @@
+discard """
+  output: '''z'''
+"""
+
+import mforwarded_pure_enum as t2
+
+echo z