import foo {.all.} reboot (#17706)

changelog.md

@@ -307,6 +307,15 @@
 - Added `iterable[T]` type class to match called iterators, which enables writing:
   `template fn(a: iterable)` instead of `template fn(a: untyped)`
 
+- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private)
+  from `foo`. It works in combination with all pre-existing import features.
+  This reduces or eliminates the need for workarounds such as using `include` (which has known issues)
+  when you need a private symbol for testing or making some internal APIs public just because
+  another internal module needs those.
+  It also helps mitigate the lack of cyclic imports in some cases.
+
+- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields
+  for an object type in the current scope.
 
 ## Compiler changes
 

compiler/ast.nim

@@ -605,6 +605,8 @@ type
 const
   routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
+  ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds
+
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
@@ -691,7 +693,7 @@ type
     mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2,
     mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
-    mSymIsInstantiationOf, mNodeId
+    mSymIsInstantiationOf, mNodeId, mPrivateAccess
 
 
 # things that we can evaluate safely at compile time, even if not asked for it:
@@ -841,6 +843,7 @@ type
     depthLevel*: int
     symbols*: TStrTable
     parent*: PScope
+    allowPrivateAccess*: seq[PSym] #  # enable access to private fields
 
   PScope* = ref TScope
 
@@ -1011,9 +1014,6 @@ const
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr,
     tyProc, tyError} # TODO
   PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM
-  ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType,
-    skIterator,
-    skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,

compiler/ic/ic.nim

@@ -36,6 +36,7 @@ type
     bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
     #producedGenerics*: Table[GenericKey, SymId]
     exports*: seq[(LitId, int32)]
+    hidden*: seq[(LitId, int32)]
     reexports*: seq[(LitId, PackedItemId)]
     compilerProcs*: seq[(LitId, int32)]
     converters*, methods*, trmacros*, pureEnums*: seq[int32]
@@ -177,6 +178,10 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
 proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
   m.imports.add toLitId(f, c, m)
 
+proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  m.hidden.add((nameId, s.itemId.item))
+
 proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   let nameId = getOrIncl(m.sh.strings, s.name.s)
   m.exports.add((nameId, s.itemId.item))
@@ -524,7 +529,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadTabSection numbersSection, m.sh.numbers
 
   loadSeqSection exportsSection, m.exports
-
+  loadSeqSection hiddenSection, m.hidden
   loadSeqSection reexportsSection, m.reexports
 
   loadSeqSection compilerProcsSection, m.compilerProcs
@@ -589,7 +594,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeTabSection numbersSection, m.sh.numbers
 
   storeSeqSection exportsSection, m.exports
-
+  storeSeqSection hiddenSection, m.hidden
   storeSeqSection reexportsSection, m.reexports
 
   storeSeqSection compilerProcsSection, m.compilerProcs
@@ -655,7 +660,9 @@ type
     syms: seq[PSym] # indexed by itemId
     types: seq[PType]
     module*: PSym # the one true module symbol.
-    iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+    iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
+      # PackedItemId so that it works with reexported symbols too
+      # ifaceHidden includes private symbols
 
   PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
 
@@ -882,12 +889,22 @@ proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
   m.iface = initTable[PIdent, seq[PackedItemId]]()
-  for e in m.fromDisk.exports:
+  m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]()
+  template impl(iface, e) =
     let nameLit = e[0]
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
+    let e2 =
-  for re in m.fromDisk.reexports:
+      when e[1] is PackedItemId: e[1]
-    let nameLit = re[0]
+      else: PackedItemId(module: LitId(0), item: e[1])
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
+    iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2)
+
+  for e in m.fromDisk.exports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.reexports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.hidden:
+    m.ifaceHidden.impl(e)
 
   let filename = AbsoluteFile toFullPath(conf, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -1053,16 +1070,21 @@ type
     values: seq[PackedItemId]
     i, module: int
 
+template interfSelect(a: LoadedModule, importHidden: bool): auto =
+  var ret = a.iface.addr
+  if importHidden: ret = a.ifaceHidden.addr
+  ret[]
+
 proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
                   g: var PackedModuleGraph; module: FileIndex;
-                  name: PIdent): PSym =
+                  name: PIdent, importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
     lastFile: FileIndex(-1),
     config: config,
     cache: cache)
-  it.values = g[int module].iface.getOrDefault(name)
+  it.values = g[int module].interfSelect(importHidden).getOrDefault(name)
   it.i = 0
   it.module = int(module)
   if it.i < it.values.len:
@@ -1070,7 +1092,7 @@ proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
     inc it.i
 
 proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
-                         g: var PackedModuleGraph; module: FileIndex): PSym =
+                         g: var PackedModuleGraph; module: FileIndex, importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
@@ -1079,7 +1101,7 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
     cache: cache)
   it.values = @[]
   it.module = int(module)
-  for v in g[int module].iface.values:
+  for v in g[int module].interfSelect(importHidden).values:
     it.values.add v
   it.i = 0
   if it.i < it.values.len:
@@ -1093,9 +1115,9 @@ proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
 
 iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
                            g: var PackedModuleGraph; module: FileIndex;
-                           name: PIdent): PSym =
+                           name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   for pid in values:
     let s = loadSym(decoder, g, int(module), pid)
     assert s != nil
@@ -1103,9 +1125,9 @@ iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
 
 proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
                       g: var PackedModuleGraph; module: FileIndex;
-                      name: PIdent): PSym =
+                      name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   result = loadSym(decoder, g, int(module), values[0])
 
 proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
@@ -1140,6 +1162,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
       echo "  ", m.sh.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
 
+    echo "hidden: " & $m.hidden.len
+    for ex in m.hidden:
+      echo "  ", m.sh.strings[ex[0]], " local ID: ", ex[1]
+
   echo "all symbols"
   for i in 0..high(m.sh.syms):
     if m.sh.syms[i].name != LitId(0):

compiler/ic/rodfiles.nim

@@ -18,6 +18,7 @@ type
     depsSection
     numbersSection
     exportsSection
+    hiddenSection
     reexportsSection
     compilerProcsSection
     trmacrosSection

compiler/importer.nim

@@ -12,7 +12,7 @@
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
   semdata, modulepaths, sigmatch, lineinfos, sets,
-  modulegraphs
+  modulegraphs, wordrecg
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -107,7 +107,25 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
+proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) =
+  template bail = globalError(c.config, n.info, "invalid pragma")
+  if n.kind == nkPragmaExpr:
+    if n.len == 2 and n[1].kind == nkPragma:
+      result[0] = n[0]
+      for ni in n[1]:
+        if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident)
+        else: bail()
+    else: bail()
+  else:
+    result[0] = n
+    if result[0].safeLen > 0:
+      (result[0][^1], result[1]) = splitPragmas(c, result[0][^1])
+
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
+  let (n, kws) = splitPragmas(c, n)
+  if kws.len > 0:
+    globalError(c.config, n.info, "unexpected pragma")
+
   let ident = lookups.considerQuotedIdent(c, n)
   let s = someSym(c.graph, fromMod, ident)
   if s == nil:
@@ -204,18 +222,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
     for i in 0..n.safeLen-1:
       importForwarded(c, n[i], exceptSet, fromMod, importSet)
 
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
   result = realModule
   c.unusedImports.add((realModule, n.info))
+  template createModuleAliasImpl(ident): untyped =
+    createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options)
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info,
+    result = createModuleAliasImpl(n[1].ident)
-                               c.config.options)
+  if importHidden:
+    if result == realModule: # avoids modifying `realModule`, see D20201209T194412.
+      result = createModuleAliasImpl(realModule.name)
+    result.options.incl optImportHidden
 
-proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
+proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
+  var ret: typeof(result)
+  proc processPragma(n2: PNode): PNode =
+    let (result2, kws) = splitPragmas(c, n2)
+    result = result2
+    for ai in kws:
+      case ai
+      of wImportHidden: ret.importHidden = true
+      else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden})
+
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    ret.node = newNodeI(nkImportAs, n.info)
+    ret.node.add n[1].processPragma
+    ret.node.add n[2]
+  else:
+    ret.node = n.processPragma
+  return ret
+
+proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
+  let transf = transformImportAs(c, n)
+  n = transf.node
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
     addImportFileDep(c, f)
@@ -232,7 +275,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
       c.recursiveDep = err
 
     discard pushOptionEntry(c)
-    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden)
     popOptionEntry(c)
 
     #echo "set back to ", L
@@ -252,16 +295,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
-proc transformImportAs(c: PContext; n: PNode): PNode =
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    result = newNodeI(nkImportAs, n.info)
-    result.add n[1]
-    result.add n[2]
-  else:
-    result = n
-
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  let it = transformImportAs(c, it)
+  var it = it
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
@@ -296,7 +331,6 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
@@ -311,7 +345,6 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)

compiler/lookups.nim

@@ -75,7 +75,7 @@ proc closeScope*(c: PContext) =
   ensureNoMissingOrUnusedSymbols(c, c.currentScope)
   rawCloseScope(c)
 
-iterator allScopes(scope: PScope): PScope =
+iterator allScopes*(scope: PScope): PScope =
   var current = scope
   while current != nil:
     yield current
@@ -311,11 +311,17 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
   if conflict != nil:
     wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
-proc addInterfaceDeclAux(c: PContext, sym: PSym) =
+from ic / ic import addHidden
-  if sfExported in sym.flags:
+
+proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) =
+  if sfExported in sym.flags or forceExport:
     # add to interface:
     if c.module != nil: exportSym(c, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
+  elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym):
+    strTableAdd(semtabAll(c.graph, c.module), sym)
+    if c.config.symbolFiles != disabledSf:
+      addHidden(c.encoder, c.packedRepr, sym)
 
 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
   addDeclAt(c, scope, sym)

compiler/magicsys.nim

@@ -50,6 +50,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
   result = g.sysTypes[kind]
   if result == nil:
     case kind
+    of tyVoid: result = sysTypeFromName("void")
     of tyInt: result = sysTypeFromName("int")
     of tyInt8: result = sysTypeFromName("int8")
     of tyInt16: result = sysTypeFromName("int16")

compiler/modulegraphs.nim

@@ -29,6 +29,7 @@ type
     patterns*: seq[LazySym]
     pureEnums*: seq[LazySym]
     interf: TStrTable
+    interfHidden: TStrTable
     uniqueName*: Rope
 
   Operators* = object
@@ -160,9 +161,25 @@ proc toBase64a(s: cstring, len: int): string =
     result.add cb64[a shr 2]
     result.add cb64[(a and 3) shl 4]
 
-template semtab*(m: PSym; g: ModuleGraph): TStrTable =
+template interfSelect(iface: Iface, importHidden: bool): TStrTable =
+  var ret = iface.interf.addr # without intermediate ptr, it creates a copy and compiler becomes 15x slower!
+  if importHidden: ret = iface.interfHidden.addr
+  ret[]
+
+template semtab(g: ModuleGraph, m: PSym): TStrTable =
   g.ifaces[m.position].interf
 
+template semtabAll*(g: ModuleGraph, m: PSym): TStrTable =
+  g.ifaces[m.position].interfHidden
+
+proc initStrTables*(g: ModuleGraph, m: PSym) =
+  initStrTable(semtab(g, m))
+  initStrTable(semtabAll(g, m))
+
+proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) =
+  strTableAdd(semtab(g, m), s)
+  strTableAdd(semtabAll(g, m), s)
+
 proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} =
   result = module < g.packed.len and g.packed[module].status == loaded
 
@@ -187,39 +204,43 @@ type
     modIndex: int
     ti: TIdentIter
     rodIt: RodIter
+    importHidden: bool
 
 proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym =
   assert m.kind == skModule
   mi.modIndex = m.position
   mi.fromRod = isCachedModule(g, mi.modIndex)
+  mi.importHidden = optImportHidden in m.options
   if mi.fromRod:
-    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name)
+    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden)
   else:
-    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name)
+    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name)
 
 proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
   if mi.fromRod:
     result = nextRodIter(mi.rodIt, g.packed)
   else:
-    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf)
+    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden))
 
 iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
+  let importHidden = optImportHidden in m.options
   if isCachedModule(g, m):
     var rodIt: RodIter
-    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position)
+    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden)
     while r != nil:
       yield r
       r = nextRodIter(rodIt, g.packed)
   else:
-    for s in g.ifaces[m.position].interf.data:
+    for s in g.ifaces[m.position].interfSelect(importHidden).data:
       if s != nil:
         yield s
 
 proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  let importHidden = optImportHidden in m.options
   if isCachedModule(g, m):
-    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name)
+    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden)
   else:
-    result = strTableGet(g.ifaces[m.position].interf, name)
+    result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
 
 proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
   result = someSym(g, g.systemModule, name)
@@ -343,26 +364,23 @@ proc hash*(u: SigHash): Hash =
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 
+template getPContext(): untyped =
+  when c is PContext: c
+  else: c.c
+
 when defined(nimfind):
   template onUse*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
+    let c = getPContext()
-      if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info)
+    if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
-    else:
-      if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
 
   template onDef*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
+    let c = getPContext()
-      if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info)
+    if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
-    else:
-      if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
 
   template onDefResolveForward*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
+    let c = getPContext()
-      if c.c.graph.onDefinitionResolveForward != nil:
+    if c.graph.onDefinitionResolveForward != nil:
-        c.c.graph.onDefinitionResolveForward(c.c.graph, s, info)
+      c.graph.onDefinitionResolveForward(c.graph, s, info)
-    else:
-      if c.graph.onDefinitionResolveForward != nil:
-        c.graph.onDefinitionResolveForward(c.graph, s, info)
 
 else:
   template onUse*(info: TLineInfo; s: PSym) = discard
@@ -392,7 +410,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
 
   g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[],
                                uniqueName: rope(uniqueModuleName(g.config, FileIndex(m.position))))
-  initStrTable(g.ifaces[m.position].interf)
+  initStrTables(g, m)
 
 proc registerModuleById*(g: ModuleGraph; m: FileIndex) =
   registerModule(g, g.packed[int m].module)

compiler/modules.nim

@@ -115,7 +115,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
   elif graph.isDirty(result):
     result.flags.excl sfDirty
     # reset module fields:
-    initStrTable(result.semtab(graph))
+    initStrTables(graph, result)
     result.ast = nil
     processModuleAux()
     graph.markClientsDirty(fileIdx)

compiler/nimeval.nim

@@ -67,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) =
   ## This can also be used to *reload* the script.
   assert i != nil
   assert i.mainModule != nil, "no main module selected"
-  initStrTable(i.mainModule.semtab(i.graph))
+  initStrTables(i.graph, i.mainModule)
   i.mainModule.ast = nil
 
   let s = if scriptStream != nil: scriptStream

compiler/options.nim

@@ -41,7 +41,7 @@ type                          # please make sure we have under 32 options
     optMemTracker,
     optSinkInference          # 'sink T' inference
     optCursorInference
-
+    optImportHidden
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**

compiler/semdata.nim

@@ -361,12 +361,12 @@ proc addPattern*(c: PContext, p: LazySym) =
     addTrmacro(c.encoder, c.packedRepr, p.sym)
 
 proc exportSym*(c: PContext; s: PSym) =
-  strTableAdd(c.module.semtab(c.graph), s)
+  strTableAdds(c.graph, c.module, s)
   if c.config.symbolFiles != disabledSf:
     addExported(c.encoder, c.packedRepr, s)
 
 proc reexportSym*(c: PContext; s: PSym) =
-  strTableAdd(c.module.semtab(c.graph), s)
+  strTableAdds(c.graph, c.module, s)
   if c.config.symbolFiles != disabledSf:
     addReexport(c.encoder, c.packedRepr, s)
 
@@ -536,6 +536,10 @@ proc checkMinSonsLen*(n: PNode, length: int; conf: ConfigRef) =
 proc isTopLevel*(c: PContext): bool {.inline.} =
   result = c.currentScope.depthLevel <= 2
 
+proc isTopLevelInsideDeclaration*(c: PContext, sym: PSym): bool {.inline.} =
+  # for routeKinds the scope isn't closed yet:
+  c.currentScope.depthLevel <= 2 + ord(sym.kind in routineKinds)
+
 proc pushCaseContext*(c: PContext, caseNode: PNode) =
   c.p.caseContext.add((caseNode, 0))
 

compiler/semmagic.nim

@@ -577,5 +577,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     if n[1].typ.skipTypes(abstractInst).kind in {tyUInt..tyUInt64}:
       n[0].sym.magic = mSubU
     result = n
+  of mPrivateAccess:
+    let sym = n[1].typ[0].sym
+    assert sym != nil
+    c.currentScope.allowPrivateAccess.add sym
+    result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid))
   else:
     result = n

compiler/semstmts.nim

@@ -1067,6 +1067,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
       elif typsym.kind == skType and sfForward in typsym.flags:
         s = typsym
         addInterfaceDecl(c, s)
+        # PRTEMP no onDef here?
       else:
         localError(c.config, name.info, typsym.name.s & " is not a type that can be forwarded")
         s = typsym

compiler/semtypes.nim

@@ -138,9 +138,10 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
       identToReplace[] = symNode
     if e.position == 0: hasNull = true
     if result.sym != nil and sfExported in result.sym.flags:
-      incl(e.flags, sfUsed)
+      incl(e.flags, {sfUsed, sfExported})
-      incl(e.flags, sfExported)
+    if result.sym != nil and not isPure:
-      if not isPure: exportSym(c, e)
+      addInterfaceDeclAux(c, e, forceExport = sfExported in result.sym.flags)
+
     result.n.add symNode
     styleCheckDef(c.config, e)
     onDef(e.info, e)

compiler/suggest.nim

@@ -253,10 +253,15 @@ proc filterSymNoOpr(s: PSym; prefix: PNode; res: var PrefixMatch): bool {.inline
 proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
   let fmoduleId = getModule(f).id
   result = sfExported in f.flags or fmoduleId == c.module.id
-  for module in c.friendModules:
+
-    if fmoduleId == module.id:
+  if not result:
-      result = true
+    for module in c.friendModules:
-      break
+      if fmoduleId == module.id: return true
+    if f.kind == skField:
+      let symObj = f.owner
+      for scope in allScopes(c.currentScope):
+        for sym in scope.allowPrivateAccess:
+          if symObj.id == sym.id: return true
 
 proc getQuality(s: PSym): range[0..100] =
   result = 100

compiler/wordrecg.nim

@@ -109,7 +109,7 @@ type
     wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr",
 
     wInOut = "inout", wByCopy = "bycopy", wByRef = "byref", wOneWay = "oneway",
-    wBitsize = "bitsize"
+    wBitsize = "bitsize", wImportHidden = "all",
 
   TSpecialWords* = set[TSpecialWord]
 

lib/std/importutils.nim

@@ -0,0 +1,34 @@
+##[
+Utilities related to import and symbol resolution.
+
+Experimental API, subject to change.
+]##
+
+#[
+Possible future APIs:
+* module symbols (https://github.com/nim-lang/Nim/pull/9560)
+* whichModule (subsumes canImport / moduleExists) (https://github.com/timotheecour/Nim/issues/376)
+* getCurrentPkgDir (https://github.com/nim-lang/Nim/pull/10530)
+* import from a computed string + related APIs (https://github.com/nim-lang/Nim/pull/10527)
+]#
+
+when defined(nimImportutilsExample):
+  type Foo = object
+    x1: int # private
+  proc initFoo*(): auto = Foo()
+
+proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} =
+  ## Enables access to private fields of `t` in current scope.
+  runnableExamples("-d:nimImportutilsExample"):
+    # here we're importing a module containing:
+    # type Foo = object
+    #   x1: int # private
+    # proc initFoo*(): auto = Foo()
+    var a = initFoo()
+    block:
+      assert not compiles(a.x1)
+      privateAccess(a.type)
+      a.x1 = 1 # accessible in this scope
+      block:
+        assert a.x1 == 1 # still in scope
+    assert not compiles(a.x1)

testament/categories.nim

@@ -500,6 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string)
 
   const tempExt = "_temp.nim"
   for it in walkDirRec(testsDir / "ic"):
+  # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test
     if isTestFile(it) and not it.endsWith(tempExt):
       let nimcache = nimcacheDir(it, options, getTestSpecTarget())
       removeDir(nimcache)

tests/ic/mimports.nim

@@ -0,0 +1,9 @@
+from mimportsb {.all.} import fnb1, hfnb3
+
+proc fn1*(): int = 1
+proc fn2*(): int = 2
+proc hfn3(): int = 3
+proc hfn4(): int = 4
+proc hfn5(): int = 5
+
+export mimportsb.fnb2, hfnb3

tests/ic/mimportsb.nim

@@ -0,0 +1,4 @@
+proc fnb1*(): int = 1
+proc fnb2*(): int = 2
+proc hfnb3(): int = 3
+proc hfnb4(): int = 4

tests/ic/timports.nim

@@ -0,0 +1,29 @@
+import mimports
+doAssert fn1() == 1
+doAssert not declared(hfn3)
+
+#!EDIT!#
+
+import mimports {.all.}
+doAssert fn1() == 1
+doAssert declared(hfn3)
+doAssert hfn3() == 3
+doAssert mimports.hfn4() == 4
+
+# reexports
+doAssert not declared(fnb1)
+doAssert not declared(hfnb4)
+doAssert fnb2() == 2
+doAssert hfnb3() == 3
+
+#!EDIT!#
+
+from mimports {.all.} import hfn3
+doAssert not declared(fn1)
+from mimports {.all.} as bar import fn1
+doAssert fn1() == 1
+doAssert hfn3() == 3
+doAssert not declared(hfn4)
+doAssert declared(mimports.hfn4)
+doAssert mimports.hfn4() == 4
+doAssert bar.hfn4() == 4

tests/importalls/m1.nim

@@ -0,0 +1,70 @@
+import ./m2
+import ./m3 {.all.} as m3
+from ./m3 as m3Bis import nil
+
+doAssert m3h2 == 2
+export m3h2
+
+export m3Bis.m3p1
+
+const foo0* = 2
+const foo1 = bar1
+
+const foo1Aux = 2
+export foo1Aux
+
+doAssert not declared(bar2)
+doAssert not compiles(bar2)
+
+var foo2 = 2
+let foo3 = 2
+
+type Foo4 = enum
+  kg1, kg2
+
+type Foo4b {.pure.} = enum
+  foo4b1, foo4b2
+
+type Foo5 = object
+  z1: string
+  z2: Foo4
+  z3: int
+  z4*: int
+
+proc `z3`*(a: Foo5): auto =
+  a.z3 * 10
+
+proc foo6(): auto = 2
+proc foo6b*(): auto = 2
+template foo7: untyped = 2
+macro foo8(): untyped = discard
+template foo9(a: int) = discard
+
+block:
+  template foo10: untyped = 2
+  type Foo11 = enum
+    kg1b, kg2b
+  proc foo12(): auto = 2
+
+proc initFoo5*(z3: int): Foo5 = Foo5(z3: z3)
+
+func foo13(): auto = 2
+iterator foo14a(): int = discard
+iterator foo14b*(): int = discard
+iterator foo14c(): int {.closure.} = discard
+iterator foo14d(): int {.inline.} = discard
+
+# fwd declare
+proc foo15(): int
+proc foo15(): int = 2
+
+proc foo16*(): int
+proc foo16(): int = 2
+
+proc foo17*(): int
+proc foo17*(): int = 2
+
+# other
+type A1 = distinct int
+type A2 = distinct int
+converter foo18(x: A1): A2 = discard

tests/importalls/m2.nim

@@ -0,0 +1,3 @@
+const bar1* = 2
+const bar2 = 2
+const bar3 = 3

tests/importalls/m3.nim

@@ -0,0 +1,5 @@
+# xxx use below naming convention in other test files: p: public, h: hidden
+const m3p1* = 2
+const m3h2 = 2
+const m3h3 = 3
+const m3h4 = 4

tests/importalls/m4.nim

@@ -0,0 +1,10 @@
+{.warning[UnusedImport]: off.} # xxx bug: this shouldn't be needed since we have `export m3`
+import ./m3 {.all.}
+import ./m3 as m3b
+export m3b
+export m3h3
+export m3.m3h4
+
+import ./m2 {.all.} as m2b
+export m2b except bar3
+

tests/importalls/mt0.nim

@@ -0,0 +1,9 @@
+import ./m1 as m
+doAssert compiles(foo0)
+doAssert not compiles(foo1)
+doAssert foo6b() == 2
+doAssert m3h2 == 2
+
+var f = initFoo5(z3=3)
+doAssert f.z3 == 30
+doAssert z3(f) == 30

tests/importalls/mt1.nim

@@ -0,0 +1,23 @@
+import ./m1 {.all.} as m
+doAssert foo1 == 2
+doAssert m.foo1 == 2
+
+doAssert m.m3h2 == 2
+doAssert m3h2 == 2
+doAssert m.foo1Aux == 2
+doAssert m.m3p1 == 2
+
+## field access
+import std/importutils
+privateAccess(Foo5)
+# var x = Foo5(z1: "foo", z2: m.kg1)
+# doAssert x.z1 == "foo"
+
+var f0: Foo5
+f0.z3 = 3
+doAssert f0.z3 == 3
+var f = initFoo5(z3=3)
+doAssert f.z3 == 3
+doAssert z3(f) == 30
+doAssert m.z3(f) == 30
+doAssert not compiles(mt1.`z3`(f)) # z3 is an imported symbol

tests/importalls/mt2.nim

@@ -0,0 +1,104 @@
+from ./m1 {.all.} as r1 import foo1
+from ./m1 {.all.} as r2 import foo7
+
+block: # different symbol kinds
+  doAssert foo1 == 2
+  doAssert r1.foo1 == 2
+  doAssert r1.foo2 == 2
+  doAssert compiles(foo1)
+  doAssert compiles(r1.foo2)
+  doAssert not compiles(foo2)
+  doAssert not compiles(m3h2)
+  doAssert r1.foo3 == 2
+
+  block: # enum
+    var a: r1.Foo4
+    let a1 = r1.kg1
+    doAssert a1 == r1.Foo4.kg1
+    type A = r1.Foo4
+    doAssert a1 == A.kg1
+    doAssert not compiles(kg1)
+    doAssert compiles(A.kg1)
+    var witness = false
+    for ai in r1.Foo4:
+      doAssert ai == a
+      doAssert ai == a1
+      witness = true
+      break
+    doAssert witness
+
+  block: # {.pure.} enum
+    var a: r1.Foo4b
+    doAssert not compiles(r1.foo4b1) # because pure
+    doAssert not compiles(foo4b1)
+    let a1 = r1.Foo4b.foo4b1
+    doAssert a1 == a
+    type A = r1.Foo4b
+    doAssert a1 == A.foo4b1
+    var witness = false
+    for ai in A:
+      doAssert ai == a
+      doAssert ai == a1
+      witness = true
+      break
+    doAssert witness
+
+  block: # object
+    doAssert compiles(r1.Foo5)
+    var a: r1.Foo5
+    doAssert compiles(a.z4)
+    doAssert not compiles(a.z3)
+
+  block: # remaining symbol kinds
+    doAssert r1.foo6() == 2
+    doAssert r1.foo6b() == 2
+    doAssert foo7() == 2
+    doAssert r2.foo6b() == 2
+
+    r1.foo8()
+    r1.foo9(1)
+    doAssert r1.foo13() == 2
+    for a in r1.foo14a(): discard
+    for a in r1.foo14b(): discard
+    for a in r1.foo14c(): discard
+    for a in r1.foo14d(): discard
+    doAssert r1.foo15() == 2
+    doAssert r1.foo16() == 2
+    doAssert r1.foo17() == 2
+    doAssert compiles(r1.foo18)
+    doAssert declared(r1.foo18)
+
+  block: # declarations at block scope should not be visible
+    doAssert declared(foo7)
+    doAssert declared(r1.foo6)
+    doAssert not declared(foo10)
+    doAssert not declared(foo6)
+    doAssert not declared(r1.Foo11)
+    doAssert not declared(r1.kg1b)
+    doAssert not declared(r1.foo12)
+    doAssert not compiles(r1.foo12())
+
+## field access
+import std/importutils
+privateAccess(r1.Foo5)
+var x = r1.Foo5(z1: "foo", z2: r1.kg1)
+doAssert x.z1 == "foo"
+
+var f0: r1.Foo5
+f0.z3 = 3
+doAssert f0.z3 == 3
+var f = r1.initFoo5(z3=3)
+doAssert f.z3 == 3
+doAssert r1.z3(f) == 30
+
+import ./m1 as r3
+doAssert not declared(foo2)
+doAssert not declared(r3.foo2)
+
+from ./m1 {.all.} as r4 import nil
+doAssert not declared(foo2)
+doAssert declared(r4.foo2)
+
+from ./m1 {.all.} import nil
+doAssert not declared(foo2)
+doAssert declared(m1.foo2)

tests/importalls/mt3.nim

@@ -0,0 +1,12 @@
+import ./m1 {.all.}
+  # D20201209T194412:here keep this as is, without `as`, so that mt8.nim test keeps
+  # checking that the original module symbol for `m1` isn't modified and that
+  # only the alias in `createModuleAlias` is affected.
+doAssert declared(m1.foo1)
+doAssert foo1 == 2
+
+
+doAssert m1.foo1 == 2
+
+doAssert not compiles(mt3.foo0) # foo0 is an imported symbol
+doAssert not compiles(mt3.foo1) # ditto

tests/importalls/mt4.nim

@@ -0,0 +1,4 @@
+import ./m1 {.all.} except foo1
+doAssert foo2 == 2
+doAssert declared(foo2)
+doAssert not compiles(foo1)

tests/importalls/mt4b.nim

@@ -0,0 +1,3 @@
+from ./m1 {.all.} as m2 import nil
+doAssert not compiles(foo1)
+doAssert m2.foo1 == 2

tests/importalls/mt5.nim

@@ -0,0 +1,9 @@
+import ./m1 {.all.} as m2 except foo1
+doAssert foo2 == 2
+doAssert not compiles(foo1)
+doAssert m2.foo1 == 2
+doAssert compiles(m2.foo1)
+
+from system {.all.} as s import ThisIsSystem
+doAssert ThisIsSystem
+doAssert s.ThisIsSystem

tests/importalls/mt6.nim

@@ -0,0 +1,13 @@
+import ./m1 {.all.} as m2
+doAssert compiles(foo1)
+doAssert compiles(m2.foo1)
+doAssert declared(foo1)
+doAssert declared(m2.foo0) # public: works fine
+
+doAssert m2.foo1 == 2
+doAssert declared(m2.foo1)
+doAssert not declared(m2.nonexistant)
+
+# also tests the quoted `""` import
+import "."/"m1" {.all.} as m1b
+doAssert compiles(m1b.foo1)

tests/importalls/mt7.nim

@@ -0,0 +1,14 @@
+include ./m1
+doAssert compiles(foo1)
+doAssert compiles(mt7.foo1)
+doAssert declared(foo1)
+doAssert declared(mt7.foo1)
+doAssert declared(mt7.foo0)
+
+var f0: Foo5
+f0.z3 = 3
+doAssert f0.z3 == 3
+var f = initFoo5(z3=3)
+doAssert f.z3 == 3
+doAssert mt7.z3(f) == 30
+doAssert z3(f) == 30

tests/importalls/mt8.nim

@@ -0,0 +1,23 @@
+#[
+test multiple imports
+]#
+
+{.warning[UnusedImport]: off.}
+import ./m1, m2 {.all.}, ./m3 {.all.}
+  # make sure this keeps using `import ./m1` without as.
+
+# m1 is regularly imported
+doAssert declared(m1.foo0)
+doAssert declared(foo0)
+
+doAssert not declared(m1.foo1)
+  # if we didn't call `createModuleAlias` even for `import f1 {.all.}`,
+  # this would fail, see D20201209T194412.
+
+# m2
+doAssert declared(m2.bar2)
+doAssert declared(bar2)
+
+# m3
+doAssert declared(m3.m3h2)
+doAssert declared(m3h2)

tests/importalls/mt9.nim

@@ -0,0 +1,12 @@
+# tests re-export of a module with import {.all.}
+
+import ./m4
+
+doAssert m3p1 == 2
+doAssert not declared(m3h2)
+doAssert m3h3 == 3
+doAssert m3h4 == 4
+
+doAssert bar1 == 2
+doAssert bar2 == 2
+doAssert not declared(bar3)

tests/importalls/tmain2.nim

@@ -0,0 +1,7 @@
+discard """
+joinable: false # for clarity, but not necessary
+"""
+
+{.warning[UnusedImport]: off.}
+# only import `mt*.nim` here; these depend on `m*.nim`
+import "."/[mt0,mt1,mt2,mt3,mt4,mt4b,mt5,mt6,mt7,mt8,mt9]