refactoring complete: explicit ident cache

compiler/cgen.nim

@@ -1174,7 +1174,7 @@ proc newModule(module: PSym): BModule =
     if (sfDeadCodeElim in module.flags):
       internalError("added pending module twice: " & module.filename)
 
-proc myOpen(module: PSym): PPassContext =
+proc myOpen(module: PSym; cache: IdentCache): PPassContext =
   result = newModule(module)
   if optGenIndex in gGlobalOptions and generatedHeader == nil:
     let f = if headerFile.len > 0: headerFile else: gProjectFull

compiler/depends.nim

@@ -46,7 +46,7 @@ proc generateDot(project: string) =
       rope(changeFileExt(extractFilename(project), "")), gDotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(module: PSym): PPassContext =
+proc myOpen(module: PSym; cache: IdentCache): PPassContext =
   var g: PGen
   new(g)
   g.module = module

compiler/docgen2.nim

@@ -49,7 +49,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
   var g = PGen(c)
   generateJson(g.doc, n)
 
-proc myOpen(module: PSym): PPassContext =
+proc myOpen(module: PSym; cache: IdentCache): PPassContext =
   var g: PGen
   new(g)
   g.module = module

compiler/guards.nim

@@ -83,10 +83,6 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
-let
-  opNot* = createMagic("not", mNot)
-  opContains* = createMagic("contains", mInSet)
-
 let
   opLe = createMagic("<=", mLeI)
   opLt = createMagic("<", mLtI)

compiler/idents.nim

@@ -112,9 +112,6 @@ proc getIdent*(self: IdentCache; identifier: string): PIdent =
 proc getIdent*(self: IdentCache; identifier: string, h: Hash): PIdent =
   result = getIdent(cstring(identifier), len(identifier), h)
 
-proc identEq*(self: IdentCache; id: PIdent, name: string): bool =
-  result = id.id == getIdent(name).id
-
 proc newIdentCache*(): IdentCache =
   if legacy.isNil:
     result = IdentCache()
@@ -135,4 +132,6 @@ proc whichKeyword*(id: PIdent): TSpecialWord =
 
 proc getIdent*(identifier: string): PIdent =
   ## for backwards compatibility.
+  if legacy.isNil:
+    discard newIdentCache()
   legacy.getIdent identifier

compiler/jsgen.nim

@@ -2276,7 +2276,7 @@ proc myOpenCached(s: PSym, rd: PRodReader): PPassContext =
   internalError("symbol files are not possible with the JS code generator")
   result = nil
 
-proc myOpen(s: PSym): PPassContext =
+proc myOpen(s: PSym; cache: IdentCache): PPassContext =
   var r = newModule(s)
   r.target = if gCmd == cmdCompileToPHP: targetPHP else: targetJS
   result = r

compiler/magicsys.nim

@@ -42,6 +42,10 @@ proc createMagic*(name: string, m: TMagic): PSym =
   result = newSym(skProc, getIdent(name), nil, unknownLineInfo())
   result.magic = m
 
+let
+  opNot* = createMagic("not", mNot)
+  opContains* = createMagic("contains", mInSet)
+
 proc getSysMagic*(name: string, m: TMagic): PSym =
   var ti: TIdentIter
   let id = getIdent(name)

compiler/main.nim

@@ -30,39 +30,39 @@ proc semanticPasses =
   registerPass verbosePass
   registerPass semPass
 
-proc commandGenDepend =
+proc commandGenDepend(cache: IdentCache) =
   semanticPasses()
   registerPass(gendependPass)
   registerPass(cleanupPass)
-  compileProject()
+  compileProject(cache)
   generateDot(gProjectFull)
   execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") &
       ' ' & changeFileExt(gProjectFull, "dot"))
 
-proc commandCheck =
+proc commandCheck(cache: IdentCache) =
   msgs.gErrorMax = high(int)  # do not stop after first error
   defineSymbol("nimcheck")
   semanticPasses()            # use an empty backend for semantic checking only
   rodPass()
-  compileProject()
+  compileProject(cache)
 
-proc commandDoc2(json: bool) =
+proc commandDoc2(cache: IdentCache; json: bool) =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
   if json: registerPass(docgen2JsonPass)
   else: registerPass(docgen2Pass)
   #registerPass(cleanupPass())
-  compileProject()
+  compileProject(cache)
   finishDoc2Pass(gProjectName)
 
-proc commandCompileToC =
+proc commandCompileToC(cache: IdentCache) =
   extccomp.initVars()
   semanticPasses()
   registerPass(cgenPass)
   rodPass()
   #registerPass(cleanupPass())
 
-  compileProject()
+  compileProject(cache)
   cgenWriteModules()
   if gCmd != cmdRun:
     extccomp.callCCompiler(changeFileExt(gProjectFull, ""))
@@ -103,7 +103,7 @@ proc commandCompileToC =
     ccgutils.resetCaches()
     GC_fullCollect()
 
-proc commandCompileToJS =
+proc commandCompileToJS(cache: IdentCache) =
   #incl(gGlobalOptions, optSafeCode)
   setTarget(osJS, cpuJS)
   #initDefines()
@@ -113,9 +113,9 @@ proc commandCompileToJS =
   if gCmd == cmdCompileToPHP: defineSymbol("nimphp")
   semanticPasses()
   registerPass(JSgenPass)
-  compileProject()
+  compileProject(cache)
 
-proc interactivePasses =
+proc interactivePasses(cache: IdentCache) =
   #incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
@@ -125,30 +125,30 @@ proc interactivePasses =
   registerPass(semPass)
   registerPass(evalPass)
 
-proc commandInteractive =
+proc commandInteractive(cache: IdentCache) =
   msgs.gErrorMax = high(int)  # do not stop after first error
-  interactivePasses()
+  interactivePasses(cache)
-  compileSystemModule()
+  compileSystemModule(cache)
   if commandArgs.len > 0:
-    discard compileModule(fileInfoIdx(gProjectFull), {})
+    discard compileModule(fileInfoIdx(gProjectFull), cache, {})
   else:
     var m = makeStdinModule()
     incl(m.flags, sfMainModule)
-    processModule(m, llStreamOpenStdIn(), nil)
+    processModule(m, llStreamOpenStdIn(), nil, cache)
 
 const evalPasses = [verbosePass, semPass, evalPass]
 
-proc evalNim(nodes: PNode, module: PSym) =
+proc evalNim(nodes: PNode, module: PSym; cache: IdentCache) =
-  carryPasses(nodes, module, evalPasses)
+  carryPasses(nodes, module, cache, evalPasses)
 
-proc commandEval(exp: string) =
+proc commandEval(cache: IdentCache; exp: string) =
   if systemModule == nil:
-    interactivePasses()
+    interactivePasses(cache)
-    compileSystemModule()
+    compileSystemModule(cache)
-  var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
+  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(echoExp.parseString, makeStdinModule())
+  evalNim(echoExp.parseString(cache), makeStdinModule(), cache)
 
-proc commandScan =
+proc commandScan(cache: IdentCache) =
   var f = addFileExt(mainCommandArg(), NimExt)
   var stream = llStreamOpen(f, fmRead)
   if stream != nil:
@@ -156,7 +156,7 @@ proc commandScan =
       L: TLexer
       tok: TToken
     initToken(tok)
-    openLexer(L, f, stream)
+    openLexer(L, f, stream, cache)
     while true:
       rawGetTok(L, tok)
       printTok(tok)
@@ -165,16 +165,16 @@ proc commandScan =
   else:
     rawMessage(errCannotOpenFile, f)
 
-proc commandSuggest =
+proc commandSuggest(cache: IdentCache) =
   if isServing:
     # XXX: hacky work-around ahead
     # Currently, it's possible to issue a idetools command, before
     # issuing the first compile command. This will leave the compiler
     # cache in a state where "no recompilation is necessary", but the
     # cgen pass was never executed at all.
-    commandCompileToC()
+    commandCompileToC(cache)
     let gDirtyBufferIdx = gTrackPos.fileIndex
-    discard compileModule(gDirtyBufferIdx, {sfDirty})
+    discard compileModule(gDirtyBufferIdx, cache, {sfDirty})
     resetModule(gDirtyBufferIdx)
   else:
     msgs.gErrorMax = high(int)  # do not stop after first error
@@ -184,7 +184,7 @@ proc commandSuggest =
     # but doesn't handle the case when it's imported module
     #var projFile = if gProjectMainIdx == gDirtyOriginalIdx: gDirtyBufferIdx
     #               else: gProjectMainIdx
-    compileProject() #(projFile)
+    compileProject(cache) #(projFile)
 
 proc resetMemory =
   resetCompilationLists()
@@ -249,28 +249,28 @@ proc mainCommand*(cache: IdentCache) =
   of "c", "cc", "compile", "compiletoc":
     # compile means compileToC currently
     gCmd = cmdCompileToC
-    commandCompileToC()
+    commandCompileToC(cache)
   of "cpp", "compiletocpp":
     gCmd = cmdCompileToCpp
     defineSymbol("cpp")
-    commandCompileToC()
+    commandCompileToC(cache)
   of "objc", "compiletooc":
     gCmd = cmdCompileToOC
     defineSymbol("objc")
-    commandCompileToC()
+    commandCompileToC(cache)
   of "run":
     gCmd = cmdRun
     when hasTinyCBackend:
       extccomp.setCC("tcc")
-      commandCompileToC()
+      commandCompileToC(cache)
     else:
       rawMessage(errInvalidCommandX, command)
   of "js", "compiletojs":
     gCmd = cmdCompileToJS
-    commandCompileToJS()
+    commandCompileToJS(cache)
   of "php":
     gCmd = cmdCompileToPHP
-    commandCompileToJS()
+    commandCompileToJS(cache)
   of "doc":
     wantMainModule()
     gCmd = cmdDoc
@@ -280,7 +280,7 @@ proc mainCommand*(cache: IdentCache) =
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     defineSymbol("nimdoc")
-    commandDoc2(false)
+    commandDoc2(cache, false)
   of "rst2html":
     gCmd = cmdRst2html
     loadConfigs(DocConfig, cache)
@@ -301,14 +301,14 @@ proc mainCommand*(cache: IdentCache) =
     loadConfigs(DocConfig, cache)
     wantMainModule()
     defineSymbol("nimdoc")
-    commandDoc2(true)
+    commandDoc2(cache, true)
   of "buildindex":
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     commandBuildIndex()
   of "gendepend":
     gCmd = cmdGenDepend
-    commandGenDepend()
+    commandGenDepend(cache)
   of "dump":
     gCmd = cmdDump
     if getConfigVar("dump.format") == "json":
@@ -337,35 +337,35 @@ proc mainCommand*(cache: IdentCache) =
       for it in iterSearchPath(searchPaths): msgWriteln(it)
   of "check":
     gCmd = cmdCheck
-    commandCheck()
+    commandCheck(cache)
   of "parse":
     gCmd = cmdParse
     wantMainModule()
-    discard parseFile(gProjectMainIdx)
+    discard parseFile(gProjectMainIdx, cache)
   of "scan":
     gCmd = cmdScan
     wantMainModule()
-    commandScan()
+    commandScan(cache)
     msgWriteln("Beware: Indentation tokens depend on the parser\'s state!")
   of "secret":
     gCmd = cmdInteractive
-    commandInteractive()
+    commandInteractive(cache)
   of "e":
     # XXX: temporary command for easier testing
-    commandEval(mainCommandArg())
+    commandEval(cache, mainCommandArg())
   of "reset":
     resetMemory()
   of "idetools":
     gCmd = cmdIdeTools
     if gEvalExpr != "":
-      commandEval(gEvalExpr)
+      commandEval(cache, gEvalExpr)
     else:
-      commandSuggest()
+      commandSuggest(cache)
   of "serve":
     isServing = true
     gGlobalOptions.incl(optCaasEnabled)
     msgs.gErrorMax = high(int)  # do not stop after first error
-    serve(mainCommand)
+    serve(cache, mainCommand)
   of "nop", "help":
     # prevent the "success" message:
     gCmd = cmdDump

compiler/modules.nim

@@ -186,14 +186,16 @@ proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
         return
     else:
       result.id = getID()
-    let validFile = processModule(result, if sfMainModule in flags and gProjectIsStdin: llStreamOpen(stdin) else: nil, rd)
+    let validFile = processModule(result,
+      if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
+      rd, cache)
     if optCaasEnabled in gGlobalOptions:
       gMemCacheData[fileIdx].compiledAt = gLastCmdTime
       gMemCacheData[fileIdx].needsRecompile = Recompiled
       if validFile: doHash fileIdx
   else:
     if checkDepMem(fileIdx) == Yes:
-      result = compileModule(fileIdx, flags)
+      result = compileModule(fileIdx, cache, flags)
     else:
       result = gCompiledModules[fileIdx]
 
@@ -208,16 +210,16 @@ proc importModule*(s: PSym, fileIdx: int32; cache: IdentCache): PSym {.procvar.}
            else: ForeignPackageNotes
 
 proc includeModule*(s: PSym, fileIdx: int32; cache: IdentCache): PNode {.procvar.} =
-  result = syntaxes.parseFile(fileIdx)
+  result = syntaxes.parseFile(fileIdx, cache)
   if optCaasEnabled in gGlobalOptions:
     growCache gMemCacheData, fileIdx
     addDep(s, fileIdx)
     doHash(fileIdx)
 
-proc compileSystemModule* =
+proc compileSystemModule*(cache: IdentCache) =
   if magicsys.systemModule == nil:
     systemFileIdx = fileInfoIdx(options.libpath/"system.nim")
-    discard compileModule(systemFileIdx, {sfSystemModule})
+    discard compileModule(systemFileIdx, cache, {sfSystemModule})
 
 proc wantMainModule* =
   if gProjectFull.len == 0:
@@ -227,15 +229,15 @@ proc wantMainModule* =
 passes.gIncludeFile = includeModule
 passes.gImportModule = importModule
 
-proc compileProject*(projectFileIdx = -1'i32) =
+proc compileProject*(cache: IdentCache; projectFileIdx = -1'i32) =
   wantMainModule()
   let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
   let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx
   if projectFile == systemFileIdx:
-    discard compileModule(projectFile, {sfMainModule, sfSystemModule})
+    discard compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
   else:
-    compileSystemModule()
+    compileSystemModule(cache)
-    discard compileModule(projectFile, {sfMainModule})
+    discard compileModule(projectFile, cache, {sfMainModule})
 
 proc makeModule*(filename: string): PSym =
   result = newModule(fileInfoIdx filename)

compiler/nim.nim

@@ -21,7 +21,7 @@ when defined(i386) and defined(windows) and defined(vcc):
 import
   commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
   extccomp, strutils, os, osproc, platform, main, parseopt, service,
-  nodejs, scriptconfig
+  nodejs, scriptconfig, idents
 
 when hasTinyCBackend:
   import tccgen
@@ -37,7 +37,7 @@ proc prependCurDir(f: string): string =
   else:
     result = f
 
-proc handleCmdLine() =
+proc handleCmdLine(cache: IdentCache) =
   if paramCount() == 0:
     writeCommandLineUsage()
   else:
@@ -61,19 +61,19 @@ proc handleCmdLine() =
     loadConfigs(DefaultConfig) # load all config files
     let scriptFile = gProjectFull.changeFileExt("nims")
     if fileExists(scriptFile):
-      runNimScript(scriptFile, freshDefines=false)
+      runNimScript(cache, scriptFile, freshDefines=false)
       # 'nim foo.nims' means to just run the NimScript file and do nothing more:
       if scriptFile == gProjectFull: return
     elif fileExists(gProjectPath / "config.nims"):
       # directory wide NimScript file
-      runNimScript(gProjectPath / "config.nims", freshDefines=false)
+      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false)
     # now process command line arguments again, because some options in the
     # command line can overwite the config file's settings
     extccomp.initVars()
     processCmdLine(passCmd2, "")
     if options.command == "":
       rawMessage(errNoCommand, command)
-    mainCommand()
+    mainCommand(cache)
     if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
     #echo(GC_getStatistics())
     if msgs.gErrorCounter == 0:
@@ -117,5 +117,5 @@ when compileOption("gc", "v2") or compileOption("gc", "refc"):
 condsyms.initDefines()
 
 when not defined(selftest):
-  handleCmdLine()
+  handleCmdLine(newIdentCache())
   msgQuit(int8(msgs.gErrorCounter > 0))

compiler/passaux.nim

@@ -10,9 +10,9 @@
 ## implements some little helper passes
 
 import
-  strutils, ast, astalgo, passes, msgs, options, idgen
+  strutils, ast, astalgo, passes, idents, msgs, options, idgen
 
-proc verboseOpen(s: PSym): PPassContext =
+proc verboseOpen(s: PSym; cache: IdentCache): PPassContext =
   #MessageOut('compiling ' + s.name.s);
   result = nil                # we don't need a context
   rawMessage(hintProcessing, s.name.s)

compiler/rodread.nim

@@ -142,7 +142,7 @@ type
     methods*: TSymSeq
     origFile: string
     inViewMode: bool
-    cache: IdentCache
+    cache*: IdentCache
 
   PRodReader* = ref TRodReader
 

compiler/scriptconfig.nim

@@ -11,7 +11,7 @@
 ## language.
 
 import
-  ast, modules, passes, passaux, condsyms,
+  ast, modules, idents, passes, passaux, condsyms,
   options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
   os, times, osproc, wordrecg, strtabs
 
@@ -25,9 +25,9 @@ proc listDirs(a: VmArgs, filter: set[PathComponent]) =
     if kind in filter: result.add path
   setResult(a, result)
 
-proc setupVM*(module: PSym; scriptName: string): PEvalContext =
+proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext =
   # For Nimble we need to export 'setupVM'.
-  result = newCtx(module)
+  result = newCtx(module, cache)
   result.mode = emRepl
   registerAdditionalOps(result)
 
@@ -134,7 +134,7 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext =
   cbconf selfExe:
     setResult(a, os.getAppFilename())
 
-proc runNimScript*(scriptName: string; freshDefines=true) =
+proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true) =
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   if freshDefines: initDefines()
@@ -148,10 +148,10 @@ proc runNimScript*(scriptName: string; freshDefines=true) =
 
   var m = makeModule(scriptName)
   incl(m.flags, sfMainModule)
-  vm.globalCtx = setupVM(m, scriptName)
+  vm.globalCtx = setupVM(m, cache, scriptName)
 
-  compileSystemModule()
+  compileSystemModule(cache)
-  discard processModule(m, llStreamOpen(scriptName, fmRead), nil)
+  discard processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)
 
   # ensure we load 'system.nim' again for the real non-config stuff!
   resetAllModulesHard()

compiler/sem.nim

@@ -398,8 +398,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
         addSon(n, prc.ast)
   c.lastGenericIdx = c.generics.len
 
-proc myOpen(module: PSym): PPassContext =
+proc myOpen(module: PSym; cache: IdentCache): PPassContext =
-  var c = newContext(module)
+  var c = newContext(module, cache)
   if c.p != nil: internalError(module.info, "sem.myOpen")
   c.semConstExpr = semConstExpr
   c.semExpr = semExpr
@@ -428,8 +428,8 @@ proc myOpen(module: PSym): PPassContext =
     gNotes = ForeignPackageNotes
   result = c
 
-proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
+proc myOpenCached(module: PSym; rd: PRodReader): PPassContext =
-  result = myOpen(module)
+  result = myOpen(module, rd.cache)
   for m in items(rd.methods): methodDef(m, true)
 
 proc isImportSystemStmt(n: PNode): bool =

compiler/semexprs.nim

@@ -620,7 +620,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
 
 proc semStaticExpr(c: PContext, n: PNode): PNode =
   let a = semExpr(c, n.sons[0])
-  result = evalStaticExpr(c.module, a, c.p.owner)
+  result = evalStaticExpr(c.module, c.cache, a, c.p.owner)
   if result.isNil:
     localError(n.info, errCannotInterpretNodeX, renderTree(n))
     result = emptyNode
@@ -861,7 +861,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent,
           s = newNodeIT(nkCurly, n.info, setType)
           for j in countup(0, sonsLen(it) - 2): addSon(s, copyTree(it.sons[j]))
           var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool))
-          addSon(inExpr, newSymNode(ast.opContains, n.info))
+          addSon(inExpr, newSymNode(opContains, n.info))
           addSon(inExpr, s)
           addSon(inExpr, copyTree(r.sons[0]))
           addSon(check, inExpr)
@@ -874,11 +874,11 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent,
             check = newNodeI(nkCheckedFieldExpr, n.info)
             addSon(check, ast.emptyNode) # make space for access node
           var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool))
-          addSon(inExpr, newSymNode(ast.opContains, n.info))
+          addSon(inExpr, newSymNode(opContains, n.info))
           addSon(inExpr, s)
           addSon(inExpr, copyTree(r.sons[0]))
           var notExpr = newNodeIT(nkCall, n.info, getSysType(tyBool))
-          addSon(notExpr, newSymNode(ast.opNot, n.info))
+          addSon(notExpr, newSymNode(opNot, n.info))
           addSon(notExpr, inExpr)
           addSon(check, notExpr)
           return

compiler/service.nim

@@ -11,7 +11,7 @@
 
 import
   times, commands, options, msgs, nimconf,
-  extccomp, strutils, os, platform, parseopt
+  extccomp, strutils, os, platform, parseopt, idents
 
 when useCaas:
   import net
@@ -45,11 +45,11 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
     if optRun notin gGlobalOptions and arguments != "" and options.command.normalize != "run":
       rawMessage(errArgsNeedRunOption, [])
 
-proc serve*(action: proc (){.nimcall.}) =
+proc serve*(cache: IdentCache; action: proc (cache: IdentCache){.nimcall.}) =
   template execute(cmd) =
     curCaasCmd = cmd
     processCmdLine(passCmd2, cmd)
-    action()
+    action(cache)
     gErrorCounter = 0
 
   let typ = getConfigVar("server.type")