new dependency tracking for nimsuggest

compiler/cgen.nim

@@ -16,6 +16,8 @@ import
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
   lowerings, semparallel
 
+from modulegraphs import ModuleGraph
+
 import strutils except `%` # collides with ropes.`%`
 
 when options.hasTinyCBackend:
@@ -1174,7 +1176,7 @@ proc newModule(module: PSym): BModule =
     if (sfDeadCodeElim in module.flags):
       internalError("added pending module twice: " & module.filename)
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   result = newModule(module)
   if optGenIndex in gGlobalOptions and generatedHeader == nil:
     let f = if headerFile.len > 0: headerFile else: gProjectFull
@@ -1209,7 +1211,7 @@ proc getCFile(m: BModule): string =
       else: ".c"
   result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext)
 
-proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
+proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
   assert optSymbolFiles in gGlobalOptions
   var m = newModule(module)
   readMergeInfo(getCFile(m), m)

compiler/depends.nim

@@ -12,6 +12,8 @@
 import
   os, options, ast, astalgo, msgs, ropes, idents, passes, importer
 
+from modulegraphs import ModuleGraph
+
 proc generateDot*(project: string)
 
 type
@@ -46,7 +48,7 @@ proc generateDot(project: string) =
       rope(changeFileExt(extractFilename(project), "")), gDotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   var g: PGen
   new(g)
   g.module = module

compiler/docgen2.nim

@@ -13,6 +13,8 @@
 import
   os, options, ast, astalgo, msgs, ropes, idents, passes, docgen
 
+from modulegraphs import ModuleGraph
+
 type
   TGen = object of TPassContext
     doc: PDoc
@@ -49,7 +51,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode =
   var g = PGen(c)
   generateJson(g.doc, n)
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   var g: PGen
   new(g)
   g.module = module

compiler/importer.nim

@@ -162,7 +162,7 @@ proc importModuleAs(n: PNode, realModule: PSym): PSym =
 proc myImportModule(c: PContext, n: PNode): PSym =
   var f = checkModuleName(n)
   if f != InvalidFileIDX:
-    result = importModuleAs(n, gImportModule(c.module, f, c.cache))
+    result = importModuleAs(n, gImportModule(c.graph, c.module, f, c.cache))
     # we cannot perform this check reliably because of
     # test: modules/import_in_config)
     if result.info.fileIndex == c.module.info.fileIndex and

compiler/jsgen.nim

@@ -35,6 +35,8 @@ import
   times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
   intsets, cgmeth, lowerings
 
+from modulegraphs import ModuleGraph
+
 type
   TTarget = enum
     targetJS, targetPHP
@@ -2272,11 +2274,11 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     for obj, content in items(globals.classes):
       genClass(obj, content, ext)
 
-proc myOpenCached(s: PSym, rd: PRodReader): PPassContext =
+proc myOpenCached(graph: ModuleGraph; s: PSym, rd: PRodReader): PPassContext =
   internalError("symbol files are not possible with the JS code generator")
   result = nil
 
-proc myOpen(s: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
   var r = newModule(s)
   r.target = if gCmd == cmdCompileToPHP: targetPHP else: targetJS
   result = r

compiler/main.nim

@@ -15,7 +15,8 @@ import
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, jsgen, json, nversion,
   platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists
+  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists,
+  modulegraphs
 
 from magicsys import systemModule, resetSysTypes
 
@@ -30,80 +31,44 @@ proc semanticPasses =
   registerPass verbosePass
   registerPass semPass
 
-proc commandGenDepend(cache: IdentCache) =
+proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) =
   semanticPasses()
   registerPass(gendependPass)
   registerPass(cleanupPass)
-  compileProject(cache)
+  compileProject(graph, cache)
   generateDot(gProjectFull)
   execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") &
       ' ' & changeFileExt(gProjectFull, "dot"))
 
-proc commandCheck(cache: IdentCache) =
+proc commandCheck(graph: ModuleGraph; cache: IdentCache) =
   msgs.gErrorMax = high(int)  # do not stop after first error
   defineSymbol("nimcheck")
   semanticPasses()            # use an empty backend for semantic checking only
   rodPass()
-  compileProject(cache)
+  compileProject(graph, cache)
 
-proc commandDoc2(cache: IdentCache; json: bool) =
+proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
   if json: registerPass(docgen2JsonPass)
   else: registerPass(docgen2Pass)
   #registerPass(cleanupPass())
-  compileProject(cache)
+  compileProject(graph, cache)
   finishDoc2Pass(gProjectName)
 
-proc commandCompileToC(cache: IdentCache) =
+proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) =
   extccomp.initVars()
   semanticPasses()
   registerPass(cgenPass)
   rodPass()
   #registerPass(cleanupPass())
 
-  compileProject(cache)
+  compileProject(graph, cache)
   cgenWriteModules()
   if gCmd != cmdRun:
     extccomp.callCCompiler(changeFileExt(gProjectFull, ""))
 
-  if isServing:
+proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) =
-    # caas will keep track only of the compilation commands
-    lastCaasCmd = curCaasCmd
-    resetCgenModules()
-    for i in 0 .. <gMemCacheData.len:
-      gMemCacheData[i].hashStatus = hashCached
-      gMemCacheData[i].needsRecompile = Maybe
-
-      # XXX: clean these global vars
-      # ccgstmts.gBreakpoints
-      # ccgthreadvars.nimtv
-      # ccgthreadvars.nimtVDeps
-      # ccgthreadvars.nimtvDeclared
-      # cgendata
-      # cgmeth?
-      # condsyms?
-      # depends?
-      # lexer.gLinesCompiled
-      # msgs - error counts
-      # magicsys, when system.nim changes
-      # rodread.rodcompilerProcs
-      # rodread.gTypeTable
-      # rodread.gMods
-
-      # !! ropes.cache
-      # semthreads.computed?
-      #
-      # suggest.usageSym
-      #
-      # XXX: can we run out of IDs?
-      # XXX: detect config reloading (implement as error/require restart)
-      # XXX: options are appended (they will accumulate over time)
-    resetCompilationLists()
-    ccgutils.resetCaches()
-    GC_fullCollect()
-
-proc commandCompileToJS(cache: IdentCache) =
   #incl(gGlobalOptions, optSafeCode)
   setTarget(osJS, cpuJS)
   #initDefines()
@@ -113,9 +78,9 @@ proc commandCompileToJS(cache: IdentCache) =
   if gCmd == cmdCompileToPHP: defineSymbol("nimphp")
   semanticPasses()
   registerPass(JSgenPass)
-  compileProject(cache)
+  compileProject(graph, cache)
 
-proc interactivePasses(cache: IdentCache) =
+proc interactivePasses(graph: ModuleGraph; cache: IdentCache) =
   #incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
@@ -125,28 +90,28 @@ proc interactivePasses(cache: IdentCache) =
   registerPass(semPass)
   registerPass(evalPass)
 
-proc commandInteractive(cache: IdentCache) =
+proc commandInteractive(graph: ModuleGraph; cache: IdentCache) =
   msgs.gErrorMax = high(int)  # do not stop after first error
-  interactivePasses(cache)
+  interactivePasses(graph, cache)
-  compileSystemModule(cache)
+  compileSystemModule(graph, cache)
   if commandArgs.len > 0:
-    discard compileModule(fileInfoIdx(gProjectFull), cache, {})
+    discard graph.compileModule(fileInfoIdx(gProjectFull), cache, {})
   else:
-    var m = makeStdinModule()
+    var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
-    processModule(m, llStreamOpenStdIn(), nil, cache)
+    processModule(graph, m, llStreamOpenStdIn(), nil, cache)
 
 const evalPasses = [verbosePass, semPass, evalPass]
 
-proc evalNim(nodes: PNode, module: PSym; cache: IdentCache) =
+proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym; cache: IdentCache) =
-  carryPasses(nodes, module, cache, evalPasses)
+  carryPasses(graph, nodes, module, cache, evalPasses)
 
-proc commandEval(cache: IdentCache; exp: string) =
+proc commandEval(graph: ModuleGraph; cache: IdentCache; exp: string) =
   if systemModule == nil:
-    interactivePasses(cache)
+    interactivePasses(graph, cache)
-    compileSystemModule(cache)
+    compileSystemModule(graph, cache)
   let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(echoExp.parseString(cache), makeStdinModule(), cache)
+  evalNim(graph, echoExp.parseString(cache), makeStdinModule(graph), cache)
 
 proc commandScan(cache: IdentCache) =
   var f = addFileExt(mainCommandArg(), NimExt)
@@ -165,75 +130,11 @@ proc commandScan(cache: IdentCache) =
   else:
     rawMessage(errCannotOpenFile, f)
 
-proc commandSuggest(cache: IdentCache) =
-  if isServing:
-    # XXX: hacky work-around ahead
-    # Currently, it's possible to issue a idetools command, before
-    # issuing the first compile command. This will leave the compiler
-    # cache in a state where "no recompilation is necessary", but the
-    # cgen pass was never executed at all.
-    commandCompileToC(cache)
-    let gDirtyBufferIdx = gTrackPos.fileIndex
-    discard compileModule(gDirtyBufferIdx, cache, {sfDirty})
-    resetModule(gDirtyBufferIdx)
-  else:
-    msgs.gErrorMax = high(int)  # do not stop after first error
-    semanticPasses()
-    rodPass()
-    # XXX: this handles the case when the dirty buffer is the main file,
-    # but doesn't handle the case when it's imported module
-    #var projFile = if gProjectMainIdx == gDirtyOriginalIdx: gDirtyBufferIdx
-    #               else: gProjectMainIdx
-    compileProject(cache) #(projFile)
-
-proc resetMemory =
-  resetCompilationLists()
-  ccgutils.resetCaches()
-  resetAllModules()
-  resetRopeCache()
-  resetSysTypes()
-  gOwners = @[]
-  resetIdentCache()
-
-  # XXX: clean these global vars
-  # ccgstmts.gBreakpoints
-  # ccgthreadvars.nimtv
-  # ccgthreadvars.nimtVDeps
-  # ccgthreadvars.nimtvDeclared
-  # cgendata
-  # cgmeth?
-  # condsyms?
-  # depends?
-  # lexer.gLinesCompiled
-  # msgs - error counts
-  # magicsys, when system.nim changes
-  # rodread.rodcompilerProcs
-  # rodread.gTypeTable
-  # rodread.gMods
-
-  # !! ropes.cache
-  #
-  # suggest.usageSym
-  #
-  # XXX: can we run out of IDs?
-  # XXX: detect config reloading (implement as error/require restart)
-  # XXX: options are appended (they will accumulate over time)
-  # vis = visimpl
-  when compileOption("gc", "v2"):
-    gcDebugging = true
-    echo "COLLECT 1"
-    GC_fullCollect()
-    echo "COLLECT 2"
-    GC_fullCollect()
-    echo "COLLECT 3"
-    GC_fullCollect()
-    echo GC_getStatistics()
-
 const
   SimulateCaasMemReset = false
   PrintRopeCacheStats = false
 
-proc mainCommand*(cache: IdentCache) =
+proc mainCommand*(graph: ModuleGraph; cache: IdentCache) =
   when SimulateCaasMemReset:
     gGlobalOptions.incl(optCaasEnabled)
 
@@ -249,28 +150,28 @@ proc mainCommand*(cache: IdentCache) =
   of "c", "cc", "compile", "compiletoc":
     # compile means compileToC currently
     gCmd = cmdCompileToC
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
   of "cpp", "compiletocpp":
     gCmd = cmdCompileToCpp
     defineSymbol("cpp")
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
   of "objc", "compiletooc":
     gCmd = cmdCompileToOC
     defineSymbol("objc")
-    commandCompileToC(cache)
+    commandCompileToC(graph, cache)
   of "run":
     gCmd = cmdRun
     when hasTinyCBackend:
       extccomp.setCC("tcc")
-      commandCompileToC(cache)
+      commandCompileToC(graph, cache)
     else:
       rawMessage(errInvalidCommandX, command)
   of "js", "compiletojs":
     gCmd = cmdCompileToJS
-    commandCompileToJS(cache)
+    commandCompileToJS(graph, cache)
   of "php":
     gCmd = cmdCompileToPHP
-    commandCompileToJS(cache)
+    commandCompileToJS(graph, cache)
   of "doc":
     wantMainModule()
     gCmd = cmdDoc
@@ -280,7 +181,7 @@ proc mainCommand*(cache: IdentCache) =
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     defineSymbol("nimdoc")
-    commandDoc2(cache, false)
+    commandDoc2(graph, cache, false)
   of "rst2html":
     gCmd = cmdRst2html
     loadConfigs(DocConfig, cache)
@@ -301,14 +202,14 @@ proc mainCommand*(cache: IdentCache) =
     loadConfigs(DocConfig, cache)
     wantMainModule()
     defineSymbol("nimdoc")
-    commandDoc2(cache, true)
+    commandDoc2(graph, cache, true)
   of "buildindex":
     gCmd = cmdDoc
     loadConfigs(DocConfig, cache)
     commandBuildIndex()
   of "gendepend":
     gCmd = cmdGenDepend
-    commandGenDepend(cache)
+    commandGenDepend(graph, cache)
   of "dump":
     gCmd = cmdDump
     if getConfigVar("dump.format") == "json":
@@ -337,7 +238,7 @@ proc mainCommand*(cache: IdentCache) =
       for it in iterSearchPath(searchPaths): msgWriteln(it)
   of "check":
     gCmd = cmdCheck
-    commandCheck(cache)
+    commandCheck(graph, cache)
   of "parse":
     gCmd = cmdParse
     wantMainModule()
@@ -346,26 +247,12 @@ proc mainCommand*(cache: IdentCache) =
     gCmd = cmdScan
     wantMainModule()
     commandScan(cache)
-    msgWriteln("Beware: Indentation tokens depend on the parser\'s state!")
+    msgWriteln("Beware: Indentation tokens depend on the parser's state!")
   of "secret":
     gCmd = cmdInteractive
-    commandInteractive(cache)
+    commandInteractive(graph, cache)
   of "e":
-    # XXX: temporary command for easier testing
+    commandEval(graph, cache, mainCommandArg())
-    commandEval(cache, mainCommandArg())
-  of "reset":
-    resetMemory()
-  of "idetools":
-    gCmd = cmdIdeTools
-    if gEvalExpr != "":
-      commandEval(cache, gEvalExpr)
-    else:
-      commandSuggest(cache)
-  of "serve":
-    isServing = true
-    gGlobalOptions.incl(optCaasEnabled)
-    msgs.gErrorMax = high(int)  # do not stop after first error
-    serve(cache, mainCommand)
   of "nop", "help":
     # prevent the "success" message:
     gCmd = cmdDump
@@ -393,4 +280,4 @@ proc mainCommand*(cache: IdentCache) =
 
   resetAttributes()
 
-proc mainCommand*() = mainCommand(newIdentCache())
+proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache())

compiler/modules.nim

@@ -7,130 +7,118 @@
 #    distribution, for details about the copyright.
 #
 
-## implements the module handling
+## Implements the module handling, including the caching of modules.
 
 import
   ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes, llstream
+  idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs
 
-type
+when false:
-  TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
+  type
-  THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
+    TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
+    THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
 
-  TModuleInMemory* = object
+    TModuleInMemory* = object
-    compiledAt*: float
+      hash*: SecureHash
-    hash*: SecureHash
+      deps*: seq[int32] ## XXX: slurped files are currently not tracked
-    deps*: seq[int32] ## XXX: slurped files are currently not tracked
-    needsRecompile*: TNeedRecompile
-    hashStatus*: THashStatus
 
-var
+      needsRecompile*: TNeedRecompile
-  gCompiledModules: seq[PSym] = @[]
+      hashStatus*: THashStatus
-  gMemCacheData*: seq[TModuleInMemory] = @[]
-    ## XXX: we should implement recycling of file IDs
-    ## if the user keeps renaming modules, the file IDs will keep growing
-  gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
-  packageSyms: TStrTable
 
-initStrTable(packageSyms)
+  var
+    gCompiledModules: seq[PSym] = @[]
+    gMemCacheData*: seq[TModuleInMemory] = @[]
+      ## XXX: we should implement recycling of file IDs
+      ## if the user keeps renaming modules, the file IDs will keep growing
+    gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why.
 
-proc getModule*(fileIdx: int32): PSym =
+  proc hashChanged(fileIdx: int32): bool =
-  if fileIdx >= 0 and fileIdx < gCompiledModules.len:
+    internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
-    result = gCompiledModules[fileIdx]
 
-proc hashChanged(fileIdx: int32): bool =
+    template updateStatus =
-  internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
+      gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
+                                         else: hashNotChanged
+      # echo "TESTING Hash: ", fileIdx.toFilename, " ", result
 
-  template updateStatus =
+    case gMemCacheData[fileIdx].hashStatus
-    gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
+    of hashHasChanged:
-                                       else: hashNotChanged
+      result = true
-    # echo "TESTING Hash: ", fileIdx.toFilename, " ", result
+    of hashNotChanged:
+      result = false
+    of hashCached:
+      let newHash = secureHashFile(fileIdx.toFullPath)
+      result = newHash != gMemCacheData[fileIdx].hash
+      gMemCacheData[fileIdx].hash = newHash
+      updateStatus()
+    of hashNotTaken:
+      gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
+      result = true
+      updateStatus()
 
-  case gMemCacheData[fileIdx].hashStatus
+  proc doHash(fileIdx: int32) =
-  of hashHasChanged:
+    if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
-    result = true
+      # echo "FIRST Hash: ", fileIdx.ToFilename
-  of hashNotChanged:
+      gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
-    result = false
-  of hashCached:
-    let newHash = secureHashFile(fileIdx.toFullPath)
-    result = newHash != gMemCacheData[fileIdx].hash
-    gMemCacheData[fileIdx].hash = newHash
-    updateStatus()
-  of hashNotTaken:
-    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
-    result = true
-    updateStatus()
 
-proc doHash(fileIdx: int32) =
+  proc resetModule*(fileIdx: int32) =
-  if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
+    # echo "HARD RESETTING ", fileIdx.toFilename
-    # echo "FIRST Hash: ", fileIdx.ToFilename
+    if fileIdx <% gMemCacheData.len:
-    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
+      gMemCacheData[fileIdx].needsRecompile = Yes
+    if fileIdx <% gCompiledModules.len:
+      gCompiledModules[fileIdx] = nil
+    if fileIdx <% cgendata.gModules.len:
+      cgendata.gModules[fileIdx] = nil
 
-proc addDep(x: PSym, dep: int32) =
+  proc resetModule*(module: PSym) =
-  growCache gMemCacheData, dep
+    let conflict = getModule(module.position.int32)
-  gMemCacheData[x.position].deps.safeAdd(dep)
+    if conflict == nil: return
+    doAssert conflict == module
+    resetModule(module.position.int32)
+    initStrTable(module.tab)
 
-proc resetModule*(fileIdx: int32) =
+  proc resetAllModules* =
-  # echo "HARD RESETTING ", fileIdx.toFilename
+    for i in 0..gCompiledModules.high:
-  if fileIdx <% gMemCacheData.len:
+      if gCompiledModules[i] != nil:
-    gMemCacheData[fileIdx].needsRecompile = Yes
+        resetModule(i.int32)
-  if fileIdx <% gCompiledModules.len:
+    resetPackageCache()
-    gCompiledModules[fileIdx] = nil
+    # for m in cgenModules(): echo "CGEN MODULE FOUND"
-  if fileIdx <% cgendata.gModules.len:
-    cgendata.gModules[fileIdx] = nil
 
-proc resetModule*(module: PSym) =
+  proc resetAllModulesHard* =
-  let conflict = getModule(module.position.int32)
+    resetPackageCache()
-  if conflict == nil: return
+    gCompiledModules.setLen 0
-  doAssert conflict == module
+    gMemCacheData.setLen 0
-  resetModule(module.position.int32)
+    magicsys.resetSysTypes()
-  initStrTable(module.tab)
+    # XXX
+    #gOwners = @[]
 
-proc resetAllModules* =
+  proc checkDepMem(fileIdx: int32): TNeedRecompile =
-  for i in 0..gCompiledModules.high:
+    template markDirty =
-    if gCompiledModules[i] != nil:
+      resetModule(fileIdx)
-      resetModule(i.int32)
+      return Yes
-  resetPackageCache()
-  initStrTable(packageSyms)
-  # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
-proc resetAllModulesHard* =
+    if gFuzzyGraphChecking:
-  resetPackageCache()
+      if gMemCacheData[fileIdx].needsRecompile != Maybe:
-  gCompiledModules.setLen 0
+        return gMemCacheData[fileIdx].needsRecompile
-  gMemCacheData.setLen 0
+    else:
-  magicsys.resetSysTypes()
+      # cycle detection: We claim that a cycle does no harm.
-  initStrTable(packageSyms)
+      if gMemCacheData[fileIdx].needsRecompile == Probing:
-  # XXX
+        return No
-  #gOwners = @[]
 
-proc checkDepMem(fileIdx: int32): TNeedRecompile =
+    if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
-  template markDirty =
+      markDirty()
-    resetModule(fileIdx)
-    return Yes
 
-  if gFuzzyGraphChecking:
+    if gMemCacheData[fileIdx].deps != nil:
-    if gMemCacheData[fileIdx].needsRecompile != Maybe:
+      gMemCacheData[fileIdx].needsRecompile = Probing
-      return gMemCacheData[fileIdx].needsRecompile
+      for dep in gMemCacheData[fileIdx].deps:
-  else:
+        let d = checkDepMem(dep)
-    # cycle detection: We claim that a cycle does no harm.
+        if d in {Yes, Recompiled}:
-    if gMemCacheData[fileIdx].needsRecompile == Probing:
+          # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
-      return No
+          markDirty()
 
-  if optForceFullMake in gGlobalOptions or hashChanged(fileIdx):
+    gMemCacheData[fileIdx].needsRecompile = No
-    markDirty()
+    return No
 
-  if gMemCacheData[fileIdx].deps != nil:
+proc newModule(graph: ModuleGraph; fileIdx: int32): PSym =
-    gMemCacheData[fileIdx].needsRecompile = Probing
-    for dep in gMemCacheData[fileIdx].deps:
-      let d = checkDepMem(dep)
-      if d in {Yes, Recompiled}:
-        # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
-        markDirty()
-
-  gMemCacheData[fileIdx].needsRecompile = No
-  return No
-
-proc newModule(fileIdx: int32): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
   new(result)
@@ -143,20 +131,19 @@ proc newModule(fileIdx: int32): PSym =
 
   result.info = newLineInfo(fileIdx, 1, 1)
   let pack = getIdent(getPackageName(filename))
-  var packSym = packageSyms.strTableGet(pack)
+  var packSym = graph.packageSyms.strTableGet(pack)
   if packSym == nil:
     let pck = getPackageName(filename)
     let pck2 = if pck.len > 0: pck else: "unknown"
     packSym = newSym(skPackage, getIdent(pck2), nil, result.info)
     initStrTable(packSym.tab)
-    packageSyms.strTableAdd(packSym)
+    graph.packageSyms.strTableAdd(packSym)
 
   result.owner = packSym
   result.position = fileIdx
 
-  growCache gMemCacheData, fileIdx
+  growCache graph.modules, fileIdx
-  growCache gCompiledModules, fileIdx
+  graph.modules[result.position] = result
-  gCompiledModules[result.position] = result
 
   incl(result.flags, sfUsed)
   initStrTable(result.tab)
@@ -167,13 +154,12 @@ proc newModule(fileIdx: int32): PSym =
   # strTableIncl() for error corrections:
   discard strTableIncl(packSym.tab, result)
 
-proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
+proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
-  result = getModule(fileIdx)
+  result = graph.getModule(fileIdx)
   if result == nil:
-    growCache gMemCacheData, fileIdx
+    #growCache gMemCacheData, fileIdx
-    gMemCacheData[fileIdx].needsRecompile = Probing
+    #gMemCacheData[fileIdx].needsRecompile = Probing
-    result = newModule(fileIdx)
+    result = newModule(graph, fileIdx)
-    #var rd = handleSymbolFile(result)
     var rd: PRodReader
     result.flags = result.flags + flags
     if sfMainModule in result.flags:
@@ -182,44 +168,51 @@ proc compileModule*(fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym =
     if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
       rd = handleSymbolFile(result, cache)
       if result.id < 0:
-        internalError("handleSymbolFile should have set the module\'s ID")
+        internalError("handleSymbolFile should have set the module's ID")
         return
     else:
       result.id = getID()
-    let validFile = processModule(result,
+    discard processModule(graph, result,
       if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
       rd, cache)
-    if optCaasEnabled in gGlobalOptions:
+    #if optCaasEnabled in gGlobalOptions:
-      gMemCacheData[fileIdx].compiledAt = gLastCmdTime
+    #  gMemCacheData[fileIdx].needsRecompile = Recompiled
-      gMemCacheData[fileIdx].needsRecompile = Recompiled
+    #  if validFile: doHash fileIdx
-      if validFile: doHash fileIdx
+  elif graph.isDirty(result):
-  else:
+    result.flags.excl sfDirty
-    if checkDepMem(fileIdx) == Yes:
+    # reset module fields:
-      result = compileModule(fileIdx, cache, flags)
+    initStrTable(result.tab)
-    else:
+    result.ast = nil
-      result = gCompiledModules[fileIdx]
+    discard processModule(graph, result,
+      if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil,
+      nil, cache)
+    graph.markClientsDirty(fileIdx)
+    when false:
+      if checkDepMem(fileIdx) == Yes:
+        result = compileModule(fileIdx, cache, flags)
+      else:
+        result = gCompiledModules[fileIdx]
 
-proc importModule*(s: PSym, fileIdx: int32; cache: IdentCache): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
+                   cache: IdentCache): PSym {.procvar.} =
   # this is called by the semantic checking phase
-  result = compileModule(fileIdx, cache, {})
+  result = compileModule(graph, fileIdx, cache, {})
-  if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
+  graph.addDep(s, fileIdx)
   #if sfSystemModule in result.flags:
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes
            else: ForeignPackageNotes
 
-proc includeModule*(s: PSym, fileIdx: int32; cache: IdentCache): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32;
+                    cache: IdentCache): PNode {.procvar.} =
   result = syntaxes.parseFile(fileIdx, cache)
-  if optCaasEnabled in gGlobalOptions:
+  graph.addDep(s, fileIdx)
-    growCache gMemCacheData, fileIdx
-    addDep(s, fileIdx)
-    doHash(fileIdx)
 
-proc compileSystemModule*(cache: IdentCache) =
+proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) =
   if magicsys.systemModule == nil:
     systemFileIdx = fileInfoIdx(options.libpath/"system.nim")
-    discard compileModule(systemFileIdx, cache, {sfSystemModule})
+    discard graph.compileModule(systemFileIdx, cache, {sfSystemModule})
 
 proc wantMainModule* =
   if gProjectFull.len == 0:
@@ -229,18 +222,19 @@ proc wantMainModule* =
 passes.gIncludeFile = includeModule
 passes.gImportModule = importModule
 
-proc compileProject*(cache: IdentCache; projectFileIdx = -1'i32) =
+proc compileProject*(graph: ModuleGraph; cache: IdentCache;
+                     projectFileIdx = -1'i32) =
   wantMainModule()
   let systemFileIdx = fileInfoIdx(options.libpath / "system.nim")
   let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx
   if projectFile == systemFileIdx:
-    discard compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
+    discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule})
   else:
-    compileSystemModule(cache)
+    graph.compileSystemModule(cache)
-    discard compileModule(projectFile, cache, {sfMainModule})
+    discard graph.compileModule(projectFile, cache, {sfMainModule})
 
-proc makeModule*(filename: string): PSym =
+proc makeModule*(graph: ModuleGraph; filename: string): PSym =
-  result = newModule(fileInfoIdx filename)
+  result = graph.newModule(fileInfoIdx filename)
   result.id = getID()
 
-proc makeStdinModule*(): PSym = makeModule"stdin"
+proc makeStdinModule*(graph: ModuleGraph): PSym = graph.makeModule"stdin"

compiler/nim.nim

@@ -14,14 +14,14 @@ when defined(gcc) and defined(windows):
     {.link: "icons/nim_icon.o".}
 
 when defined(amd64) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-amd64-windows-vcc.res" .}
+  {.link: "icons/nim-amd64-windows-vcc.res".}
 when defined(i386) and defined(windows) and defined(vcc):
-  {.link: "icons/nim-i386-windows-vcc.res" .}
+  {.link: "icons/nim-i386-windows-vcc.res".}
 
 import
   commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
   extccomp, strutils, os, osproc, platform, main, parseopt, service,
-  nodejs, scriptconfig, idents
+  nodejs, scriptconfig, idents, modulegraphs
 
 when hasTinyCBackend:
   import tccgen
@@ -73,7 +73,7 @@ proc handleCmdLine(cache: IdentCache) =
     processCmdLine(passCmd2, "")
     if options.command == "":
       rawMessage(errNoCommand, command)
-    mainCommand(cache)
+    mainCommand(newModuleGraph(), cache)
     if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
     #echo(GC_getStatistics())
     if msgs.gErrorCounter == 0:

compiler/nimsuggest/nimsuggest.nim

@@ -1,12 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Nimsuggest has been moved to https://github.com/nim-lang/nimsuggest
-
-{.error: "This project has moved to the following repo: https://github.com/nim-lang/nimsuggest".}

compiler/passaux.nim

@@ -12,7 +12,9 @@
 import
   strutils, ast, astalgo, passes, idents, msgs, options, idgen
 
-proc verboseOpen(s: PSym; cache: IdentCache): PPassContext =
+from modulegraphs import ModuleGraph
+
+proc verboseOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext =
   #MessageOut('compiling ' + s.name.s);
   result = nil                # we don't need a context
   rawMessage(hintProcessing, s.name.s)

compiler/passes.nim

@@ -13,7 +13,7 @@
 import
   strutils, lists, options, ast, astalgo, llstream, msgs, platform, os,
   condsyms, idents, renderer, types, extccomp, math, magicsys, nversion,
-  nimsets, syntaxes, times, rodread, idgen
+  nimsets, syntaxes, times, rodread, idgen, modulegraphs
 
 type
   TPassContext* = object of RootObj # the pass's context
@@ -21,9 +21,9 @@ type
 
   PPassContext* = ref TPassContext
 
-  TPassOpen* = proc (module: PSym; cache: IdentCache): PPassContext {.nimcall.}
+  TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.}
   TPassOpenCached* =
-    proc (module: PSym, rd: PRodReader): PPassContext {.nimcall.}
+    proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.}
   TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.}
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
@@ -48,8 +48,8 @@ proc makePass*(open: TPassOpen = nil,
 
 # the semantic checker needs these:
 var
-  gImportModule*: proc (m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.}
+  gImportModule*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.}
-  gIncludeFile*: proc (m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.}
+  gIncludeFile*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.}
 
 # implementation
 
@@ -90,29 +90,32 @@ proc registerPass*(p: TPass) =
   gPasses[gPassesLen] = p
   inc(gPassesLen)
 
-proc carryPass*(p: TPass, module: PSym; cache: IdentCache;
+proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache;
                 m: TPassData): TPassData =
-  var c = p.open(module, cache)
+  var c = p.open(g, module, cache)
   result.input = p.process(c, m.input)
   result.closeOutput = if p.close != nil: p.close(c, m.closeOutput)
                        else: m.closeOutput
 
-proc carryPasses*(nodes: PNode, module: PSym; cache: IdentCache; passes: TPasses) =
+proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym;
+                  cache: IdentCache; passes: TPasses) =
   var passdata: TPassData
   passdata.input = nodes
   for pass in passes:
-    passdata = carryPass(pass, module, cache, passdata)
+    passdata = carryPass(g, pass, module, cache, passdata)
 
-proc openPasses(a: var TPassContextArray, module: PSym; cache: IdentCache) =
+proc openPasses(g: ModuleGraph; a: var TPassContextArray;
+                module: PSym; cache: IdentCache) =
   for i in countup(0, gPassesLen - 1):
     if not isNil(gPasses[i].open):
-      a[i] = gPasses[i].open(module, cache)
+      a[i] = gPasses[i].open(g, module, cache)
     else: a[i] = nil
 
-proc openPassesCached(a: var TPassContextArray, module: PSym, rd: PRodReader) =
+proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym,
+                      rd: PRodReader) =
   for i in countup(0, gPassesLen - 1):
     if not isNil(gPasses[i].openCached):
-      a[i] = gPasses[i].openCached(module, rd)
+      a[i] = gPasses[i].openCached(g, module, rd)
       if a[i] != nil:
         a[i].fromCache = true
     else:
@@ -155,7 +158,7 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
     importStmt.addSon str
     if not processTopLevelStmt(importStmt, a): break
 
-proc processModule*(module: PSym, stream: PLLStream,
+proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream,
                     rd: PRodReader; cache: IdentCache): bool {.discardable.} =
   var
     p: TParsers
@@ -163,7 +166,7 @@ proc processModule*(module: PSym, stream: PLLStream,
     s: PLLStream
     fileIdx = module.fileIdx
   if rd == nil:
-    openPasses(a, module, cache)
+    openPasses(graph, a, module, cache)
     if stream == nil:
       let filename = fileIdx.toFullPathConsiderDirty
       s = llStreamOpen(filename, fmRead)
@@ -203,7 +206,7 @@ proc processModule*(module: PSym, stream: PLLStream,
     # id synchronization point for more consistent code generation:
     idSynchronizationPoint(1000)
   else:
-    openPassesCached(a, module, rd)
+    openPassesCached(graph, a, module, rd)
     var n = loadInitSection(rd)
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)

compiler/rodwrite.nim

@@ -16,6 +16,8 @@ import
   condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
   rodutils
 
+from modulegraphs import ModuleGraph
+
 type
   TRodWriter = object of TPassContext
     module: PSym
@@ -633,7 +635,7 @@ proc process(c: PPassContext, n: PNode): PNode =
   else:
     discard
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   if module.id < 0: internalError("rodwrite: module ID not set")
   var w = newRodWriter(module.fileIdx.getHash, module, cache)
   rawAddInterfaceSym(w, module)

compiler/scriptconfig.nim

@@ -13,7 +13,7 @@
 import
   ast, modules, idents, passes, passaux, condsyms,
   options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
-  os, times, osproc, wordrecg, strtabs
+  os, times, osproc, wordrecg, strtabs, modulegraphs
 
 # we support 'cmpIgnoreStyle' natively for efficiency:
 from strutils import cmpIgnoreStyle, contains
@@ -134,9 +134,11 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext
   cbconf selfExe:
     setResult(a, os.getAppFilename())
 
-proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true) =
+proc runNimScript*(cache: IdentCache; scriptName: string;
+                   freshDefines=true) =
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
+  let graph = newModuleGraph()
   if freshDefines: initDefines()
 
   defineSymbol("nimscript")
@@ -146,15 +148,15 @@ proc runNimScript*(cache: IdentCache; scriptName: string; freshDefines=true) =
 
   appendStr(searchPaths, options.libpath)
 
-  var m = makeModule(scriptName)
+  var m = graph.makeModule(scriptName)
   incl(m.flags, sfMainModule)
   vm.globalCtx = setupVM(m, cache, scriptName)
 
-  compileSystemModule(cache)
+  graph.compileSystemModule(cache)
-  discard processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)
+  discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache)
 
   # ensure we load 'system.nim' again for the real non-config stuff!
-  resetAllModulesHard()
+  #resetAllModulesHard()
   vm.globalCtx = nil
   # do not remove the defined symbols
   #initDefines()

compiler/sem.nim

@@ -18,6 +18,8 @@ import
   evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity,
   semparallel, lowerings, pluginsupport, plugins.active
 
+from modulegraphs import ModuleGraph
+
 when defined(nimfix):
   import nimfix.prettybase
 
@@ -398,8 +400,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
         addSon(n, prc.ast)
   c.lastGenericIdx = c.generics.len
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
-  var c = newContext(module, cache)
+  var c = newContext(graph, module, cache)
   if c.p != nil: internalError(module.info, "sem.myOpen")
   c.semConstExpr = semConstExpr
   c.semExpr = semExpr
@@ -428,8 +430,8 @@ proc myOpen(module: PSym; cache: IdentCache): PPassContext =
     gNotes = ForeignPackageNotes
   result = c
 
-proc myOpenCached(module: PSym; rd: PRodReader): PPassContext =
+proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext =
-  result = myOpen(module, rd.cache)
+  result = myOpen(graph, module, rd.cache)
   for m in items(rd.methods): methodDef(m, true)
 
 proc isImportSystemStmt(n: PNode): bool =

compiler/semdata.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2016 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -13,7 +13,8 @@ import
   strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
   wordrecg,
   ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math,
-  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef
+  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef,
+  modulegraphs
 
 type
   TOptionEntry* = object of lists.TListEntry # entries to put on a
@@ -107,6 +108,7 @@ type
                             op: TTypeAttachedOp; col: int): PSym {.nimcall.}
     selfName*: PIdent
     cache*: IdentCache
+    graph*: ModuleGraph
     signatures*: TStrTable
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
@@ -151,7 +153,7 @@ proc newOptionEntry*(): POptionEntry =
   result.dynlib = nil
   result.notes = gNotes
 
-proc newContext*(module: PSym; cache: IdentCache): PContext =
+proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext =
   new(result)
   result.ambiguousSymbols = initIntSet()
   initLinkedList(result.optionStack)
@@ -166,6 +168,7 @@ proc newContext*(module: PSym; cache: IdentCache): PContext =
   result.generics = @[]
   result.unknownIdents = initIntSet()
   result.cache = cache
+  result.graph = graph
   initStrTable(result.signatures)
 
 

compiler/semstmts.nim

@@ -820,7 +820,7 @@ proc semAllTypeSections(c: PContext; n: PNode): PNode =
           if containsOrIncl(c.includedFiles, f):
             localError(n.info, errRecursiveDependencyX, f.toFilename)
           else:
-            let code = gIncludeFile(c.module, f, c.cache)
+            let code = gIncludeFile(c.graph, c.module, f, c.cache)
             gatherStmts c, code, result
             excl(c.includedFiles, f)
     of nkStmtList:
@@ -1418,7 +1418,7 @@ proc evalInclude(c: PContext, n: PNode): PNode =
       if containsOrIncl(c.includedFiles, f):
         localError(n.info, errRecursiveDependencyX, f.toFilename)
       else:
-        addSon(result, semStmt(c, gIncludeFile(c.module, f, c.cache)))
+        addSon(result, semStmt(c, gIncludeFile(c.graph, c.module, f, c.cache)))
         excl(c.includedFiles, f)
 
 proc setLine(n: PNode, info: TLineInfo) =

compiler/suggest.nim

@@ -233,7 +233,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
         # error: no known module name:
         typ = nil
       else:
-        let m = gImportModule(c.module, fullpath.fileInfoIdx, c.cache)
+        let m = gImportModule(c.graph, c.module, fullpath.fileInfoIdx, c.cache)
         if m == nil: typ = nil
         else:
           for it in items(n.sym.tab):

compiler/vm.nim

@@ -24,6 +24,8 @@ import
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate
 
+from modulegraphs import ModuleGraph
+
 when hasFFI:
   import evalffi
 
@@ -1516,7 +1518,7 @@ proc setupGlobalCtx(module: PSym; cache: IdentCache) =
   else:
     refresh(globalCtx, module)
 
-proc myOpen(module: PSym; cache: IdentCache): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext =
   #var c = newEvalContext(module, emRepl)
   #c.features = {allowCast, allowFFI, allowInfiniteLoops}
   #pushStackFrame(c, newStackFrame())

doc/basicopt.txt

@@ -5,7 +5,7 @@
 Command:
   //compile, c                compile project with default code generator (C)
   //doc                       generate the documentation for inputfile
-  //doc2                      generate the documentation for the whole project
+  //doc2                      generate the documentation for inputfile
 
 Arguments:
   arguments are passed to the program being run (if --run option is selected)

tools/nimsuggest/nimsuggest.nim

@@ -17,7 +17,7 @@ import compiler/options, compiler/commands, compiler/modules, compiler/sem,
   compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf,
   compiler/extccomp, compiler/condsyms, compiler/lists,
   compiler/sigmatch, compiler/ast, compiler/scriptconfig,
-  compiler/idents
+  compiler/idents, compiler/modulegraphs
 
 when defined(windows):
   import winlean
@@ -128,19 +128,19 @@ proc findNode(n: PNode): PSym =
       let res = n.sons[i].findNode
       if res != nil: return res
 
-proc symFromInfo(gTrackPos: TLineInfo): PSym =
+proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym =
-  let m = getModule(gTrackPos.fileIndex)
+  let m = graph.getModule(gTrackPos.fileIndex)
   #echo m.isNil, " I knew it ", gTrackPos.fileIndex
   if m != nil and m.ast != nil:
     result = m.ast.findNode
 
 proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
-             cache: IdentCache) =
+             graph: ModuleGraph; cache: IdentCache) =
   if gLogging:
     logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]")
   gIdeCmd = cmd
   if cmd == ideUse and suggestVersion != 2:
-    modules.resetAllModules()
+    graph.resetAllModules()
   var isKnownFile = true
   let dirtyIdx = file.fileInfoIdx(isKnownFile)
 
@@ -152,23 +152,26 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
   if suggestVersion < 2:
     usageSym = nil
   if not isKnownFile:
-    compileProject(cache)
+    graph.compileProject(cache)
   if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and
       dirtyfile.len == 0:
     discard "no need to recompile anything"
   else:
-    resetModule dirtyIdx
+    #resetModule dirtyIdx
-    if dirtyIdx != gProjectMainIdx:
+    #if dirtyIdx != gProjectMainIdx:
-      resetModule gProjectMainIdx
+    #  resetModule gProjectMainIdx
-    compileProject(cache, dirtyIdx)
+    graph.markDirty dirtyIdx
+    graph.markClientsDirty dirtyIdx
+    graph.compileProject(cache, dirtyIdx)
   if gIdeCmd in {ideUse, ideDus}:
-    let u = if suggestVersion >= 2: symFromInfo(gTrackPos) else: usageSym
+    let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: usageSym
     if u != nil:
       listUsages(u)
     else:
       localError(gTrackPos, "found no symbol at this position " & $gTrackPos)
 
-proc executeEpc(cmd: IdeCmd, args: SexpNode; cache: IdentCache) =
+proc executeEpc(cmd: IdeCmd, args: SexpNode;
+                graph: ModuleGraph; cache: IdentCache) =
   let
     file = args[0].getStr
     line = args[1].getNum
@@ -176,7 +179,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode; cache: IdentCache) =
   var dirtyfile = ""
   if len(args) > 3:
     dirtyfile = args[3].getStr(nil)
-  execute(cmd, file, dirtyfile, int(line), int(column), cache)
+  execute(cmd, file, dirtyfile, int(line), int(column), graph, cache)
 
 proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string,
                return_symbol = "return") =
@@ -192,7 +195,7 @@ template sendEpc(results: typed, tdef, hook: untyped) =
       else: s
     )
 
-  executeEpc(gIdeCmd, args, cache)
+  executeEpc(gIdeCmd, args, graph, cache)
   let res = sexp(results)
   if gLogging:
     logStr($res)
@@ -215,7 +218,7 @@ proc connectToNextFreePort(server: Socket, host: string): Port =
   let (_, port) = server.getLocalAddr
   result = port
 
-proc parseCmdLine(cmd: string; cache: IdentCache) =
+proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) =
   template toggle(sw) =
     if sw in gGlobalOptions:
       excl(gGlobalOptions, sw)
@@ -256,25 +259,25 @@ proc parseCmdLine(cmd: string; cache: IdentCache) =
   i += skipWhile(cmd, seps, i)
   i += parseInt(cmd, col, i)
 
-  execute(gIdeCmd, orig, dirtyfile, line, col-1, cache)
+  execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache)
 
-proc serveStdin(cache: IdentCache) =
+proc serveStdin(graph: ModuleGraph; cache: IdentCache) =
   if gEmitEof:
     echo DummyEof
     while true:
       let line = readLine(stdin)
-      parseCmdLine line, cache
+      parseCmdLine line, graph, cache
       echo DummyEof
       flushFile(stdout)
   else:
     echo Help
     var line = ""
     while readLineFromStdin("> ", line):
-      parseCmdLine line, cache
+      parseCmdLine line, graph, cache
       echo ""
       flushFile(stdout)
 
-proc serveTcp(cache: IdentCache) =
+proc serveTcp(graph: ModuleGraph; cache: IdentCache) =
   var server = newSocket()
   server.bindAddr(gPort, gAddress)
   var inp = "".TaintedString
@@ -288,12 +291,12 @@ proc serveTcp(cache: IdentCache) =
     accept(server, stdoutSocket)
 
     stdoutSocket.readLine(inp)
-    parseCmdLine inp.string, cache
+    parseCmdLine inp.string, graph, cache
 
     stdoutSocket.send("\c\L")
     stdoutSocket.close()
 
-proc serveEpc(server: Socket; cache: IdentCache) =
+proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) =
   var client = newSocket()
   # Wait for connection
   accept(server, client)
@@ -346,11 +349,7 @@ proc serveEpc(server: Socket; cache: IdentCache) =
                          "unexpected call: " & epcAPI
       raise newException(EUnexpectedCommand, errMessage)
 
-template beCompatible() =
+proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
-  when compiles(modules.gFuzzyGraphChecking):
-    modules.gFuzzyGraphChecking = true
-
-proc mainCommand(cache: IdentCache) =
   clearPasses()
   registerPass verbosePass
   registerPass semPass
@@ -368,26 +367,23 @@ proc mainCommand(cache: IdentCache) =
 
   case gMode
   of mstdin:
-    beCompatible()
+    compileProject(graph, cache)
-    compileProject(cache)
     #modules.gFuzzyGraphChecking = false
-    serveStdin(cache)
+    serveStdin(graph, cache)
   of mtcp:
     # until somebody accepted the connection, produce no output (logging is too
     # slow for big projects):
     msgs.writelnHook = proc (msg: string) = discard
-    beCompatible()
+    compileProject(graph, cache)
-    compileProject(cache)
     #modules.gFuzzyGraphChecking = false
-    serveTcp(cache)
+    serveTcp(graph, cache)
   of mepc:
-    beCompatible()
     var server = newSocket()
     let port = connectToNextFreePort(server, "localhost")
     server.listen()
     echo port
-    compileProject(cache)
+    compileProject(graph, cache)
-    serveEpc(server, cache)
+    serveEpc(server, graph, cache)
 
 proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
   var p = parseopt.initOptParser(cmd)
@@ -464,7 +460,9 @@ proc handleCmdLine(cache: IdentCache) =
     extccomp.initVars()
     processCmdLine(passCmd2, "")
 
-    mainCommand(cache)
+    let graph = newModuleGraph()
+    graph.suggestMode = true
+    mainCommand(graph, cache)
 
 when false:
   proc quitCalled() {.noconv.} =

tools/nimsuggest/tester.nim

@@ -27,7 +27,7 @@ proc parseTest(filename: string): Test =
   for x in lines(filename):
     let marker = x.find(cursorMarker)+1
     if marker > 0:
-      markers.add filename & ";" & dest & ":" & $i & ":" & $marker
+      markers.add "\"" & filename & "\";\"" & dest & "\":" & $i & ":" & $marker
       tmp.writeLine x.replace(cursorMarker, "")
     else:
       tmp.writeLine x