replaces implicit passes array registed at runtime with explicit function calls; simplify compilation pipeline (#21444)

* abolish using passes in the compiler; simplify compilation pipeline

* duplicate code

* Really cool to have the same signature...

* haul

* unify other backends

* refactor process

* introduce PipelinePhase

* refactor compiler

* fixes passes

* fixes nimsuggest

* add a sentinel

* enable docs checkj

* activate doc testing

* clean up

* complete cleanups

compiler/cgen.nim

@@ -12,11 +12,13 @@
 import
   ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
   nversion, nimsets, msgs, bitsets, idents, types,
-  ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
+  ccgutils, os, ropes, math, wordrecg, treetab, cgmeth,
   rodutils, renderer, cgendata, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
   injectdestructors, astmsgs, modulepaths, backendpragmas
 
+import pipelineutils
+
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
@@ -1938,8 +1940,7 @@ template injectG() {.dirty.} =
     graph.backend = newModuleList(graph)
   let g = BModuleList(graph.backend)
 
-
+proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
   injectG()
   result = newModule(g, module, graph.config)
   result.idgen = idgen
@@ -2007,7 +2008,7 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) =
 
 proc genTopLevelStmt*(m: BModule; n: PNode) =
   ## Also called from `ic/cbackend.nim`.
-  if passes.skipCodegen(m.config, n): return
+  if pipelineutils.skipCodegen(m.config, n): return
   m.initProc.options = initProcOptions(m)
   #softRnl = if optLineDir in m.config.options: noRnl else: rnl
   # XXX replicate this logic!
@@ -2020,12 +2021,6 @@ proc genTopLevelStmt*(m: BModule; n: PNode) =
   else:
     genProcBody(m.initProc, transformedN)
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
-  result = n
-  if b != nil:
-    var m = BModule(b)
-    genTopLevelStmt(m, n)
-
 proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
   if optForceFullMake notin m.config.globalOptions:
     if not moduleHasChanged(m.g.graph, m.module):
@@ -2102,7 +2097,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
     if {optGenStaticLib, optGenDynLib, optNoMain} * m.config.globalOptions == {}:
       for i in countdown(high(graph.globalDestructors), 0):
         n.add graph.globalDestructors[i]
-  if passes.skipCodegen(m.config, n): return
+  if pipelineutils.skipCodegen(m.config, n): return
   if moduleHasChanged(graph, m.module):
     # if the module is cached, we don't regenerate the main proc
     # nor the dispatchers? But if the dispatchers changed?
@@ -2143,12 +2138,6 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
   let mm = m
   m.g.modulesClosed.add mm
 
-
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
-  result = n
-  if b == nil: return
-  finalCodegenActions(graph, BModule(b), n)
-
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given
   # a second pass here
@@ -2175,5 +2164,3 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
     m.writeModule(pending=true)
   writeMapping(config, g.mapping)
   if g.generatedHeader != nil: writeHeader(g.generatedHeader)
-
-const cgenPass* = makePass(myOpen, myProcess, myClose)

compiler/depends.nim

@@ -9,7 +9,7 @@
 
 # This module implements a dependency file generator.
 
-import options, ast, ropes, passes, pathutils, msgs, lineinfos
+import options, ast, ropes, pathutils, msgs, lineinfos
 
 import modulegraphs
 
@@ -79,7 +79,7 @@ proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
   let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
   addDependencyAux(b, parent, child)
 
-proc addDotDependency(c: PPassContext, n: PNode): PNode =
+proc addDotDependency*(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
   let b = Backend(g.graph.backend)
@@ -100,7 +100,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module
@@ -109,6 +109,3 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
   if graph.backend == nil:
     graph.backend = Backend(dotGraph: "")
   result = g
-
-const gendependPass* = makePass(open = myOpen, process = addDotDependency)
-

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils, packages
+  options, ast, msgs, docgen, lineinfos, pathutils, packages
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -38,21 +38,21 @@ template closeImpl(body: untyped) {.dirty.} =
     except IOError:
       discard
 
-proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutput(g.doc, useWarning, groupedToc)
 
-proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
+proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
     writeOutputJson(g.doc, useWarning)
 
-proc processNode(c: PPassContext, n: PNode): PNode =
+proc processNode*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
     generateDoc(g.doc, n, n, g.config)
 
-proc processNodeJson(c: PPassContext, n: PNode): PNode =
+proc processNodeJson*(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
@@ -68,20 +68,11 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.doc = d
   result = g
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(HtmlExt)
 
-proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(TexExt)
 
-proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
+proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext =
   myOpenImpl(JsonExt)
-
-const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-const docgen2TexPass* = makePass(open = myOpenTex, process = processNode,
-                                 close = close)
-const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson,
-                                  close = closeJson)
-
-proc finishDoc2Pass*(project: string) =
-  discard

compiler/jsgen.nim

@@ -31,10 +31,12 @@ implements the required case distinction.
 import
   ast, trees, magicsys, options,
   nversion, msgs, idents, types,
-  ropes, passes, ccgutils, wordrecg, renderer,
+  ropes, ccgutils, wordrecg, renderer,
   cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils,
   transf, injectdestructors, sourcemap, astmsgs, backendpragmas
 
+import pipelineutils
+
 import json, sets, math, tables, intsets
 import strutils except addf
 
@@ -2830,11 +2832,11 @@ proc genModule(p: PProc, n: PNode) =
   if optStackTrace in p.options:
     p.body.add(frameDestroy(p))
 
-proc myProcess(b: PPassContext, n: PNode): PNode =
+proc processJSCodeGen*(b: PPassContext, n: PNode): PNode =
   ## Generate JS code for a node.
   result = n
   let m = BModule(b)
-  if passes.skipCodegen(m.config, n): return n
+  if pipelineutils.skipCodegen(m.config, n): return n
   if m.module == nil: internalError(m.config, n.info, "myProcess")
   let globals = PGlobals(m.graph.backend)
   var p = newInitProc(globals, m)
@@ -2869,7 +2871,7 @@ proc getClassName(t: PType): Rope =
   if s.loc.r != "": result = s.loc.r
   else: result = rope(s.name.s)
 
-proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
+proc finalJSCodeGen*(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   ## Finalize JS code generation of a Nim module.
   ## Param `n` may contain nodes returned from the last module close call.
   var m = BModule(b)
@@ -2879,14 +2881,14 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     for i in countdown(high(graph.globalDestructors), 0):
       n.add graph.globalDestructors[i]
   # Process any nodes left over from the last call to `myClose`.
-  result = myProcess(b, n)
+  result = processJSCodeGen(b, n)
   # Some codegen is different (such as no stacktraces; see `initProcOptions`)
   # when `std/system` is being processed.
   if sfSystemModule in m.module.flags:
     PGlobals(graph.backend).inSystem = false
   # Check if codegen should continue before any files are generated.
   # It may bail early is if too many errors have been raised.
-  if passes.skipCodegen(m.config, n): return n
+  if pipelineutils.skipCodegen(m.config, n): return n
   # Nim modules are compiled into a single JS file.
   # If this is the main module, then this is the final call to `myClose`.
   if sfMainModule in m.module.flags:
@@ -2904,9 +2906,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       if not writeRope(code, outFile):
         rawMessage(m.config, errCannotOpenFile, outFile.string)
 
-proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext =
+proc setupJSgen*(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext =
-  ## Create the JS backend pass context `BModule` for a Nim module.
   result = newModule(graph, s)
   result.idgen = idgen
-
-const JSgenPass* = makePass(myOpen, myProcess, myClose)

compiler/main.nim

@@ -16,9 +16,9 @@ import
   std/[strutils, os, times, tables, sha1, with, json],
   llstream, ast, lexer, syntaxes, options, msgs,
   condsyms,
-  sem, idents, passes, extccomp,
+  idents, extccomp,
   cgen, nversion,
-  platform, nimconf, passaux, depends, vm,
+  platform, nimconf, depends,
   modules,
   modulegraphs, lineinfos, pathutils, vmprofiler
 
@@ -29,12 +29,10 @@ when defined(nimPreviewSlimSystem):
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer
 
-when not defined(leanCompiler):
+import pipelines
-  import jsgen, docgen, docgen2
 
-proc semanticPasses(g: ModuleGraph) =
+when not defined(leanCompiler):
-  registerPass g, verbosePass
+  import docgen
-  registerPass g, semPass
 
 proc writeDepsFile(g: ModuleGraph) =
   let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps")
@@ -68,9 +66,8 @@ proc writeCMakeDepsFile(conf: ConfigRef) =
     fl.close()
 
 proc commandGenDepend(graph: ModuleGraph) =
-  semanticPasses(graph)
+  setPipeLinePass(graph, GenDependPass)
-  registerPass(graph, gendependPass)
+  compilePipelineProject(graph)
-  compileProject(graph)
   let project = graph.config.projectFull
   writeDepsFile(graph)
   generateDot(graph, project)
@@ -87,8 +84,8 @@ proc commandCheck(graph: ModuleGraph) =
     defineSymbol(conf.symbols, "nimconfig")
   elif conf.backend == backendJs:
     setTarget(conf.target, osJS, cpuJS)
-  semanticPasses(graph)  # use an empty backend for semantic checking only
+  setPipeLinePass(graph, SemPass)
-  compileProject(graph)
+  compilePipelineProject(graph)
 
   if conf.symbolFiles != disabledSf:
     case conf.ideCmd
@@ -102,22 +99,20 @@ when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; ext: string) =
     handleDocOutputOptions graph.config
     graph.config.setErrorMaxHighMaybe
-    semanticPasses(graph)
     case ext:
-    of TexExt:  registerPass(graph, docgen2TexPass)
+    of TexExt:
-    of JsonExt: registerPass(graph, docgen2JsonPass)
+      setPipeLinePass(graph, Docgen2TexPass)
-    of HtmlExt: registerPass(graph, docgen2Pass)
+    of JsonExt:
+      setPipeLinePass(graph, Docgen2JsonPass)
+    of HtmlExt:
+      setPipeLinePass(graph, Docgen2Pass)
     else: doAssert false, $ext
-    compileProject(graph)
+    compilePipelineProject(graph)
-    finishDoc2Pass(graph.config.projectName)
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
   extccomp.initVars(conf)
-  semanticPasses(graph)
   if conf.symbolFiles == disabledSf:
-    registerPass(graph, cgenPass)
-
     if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
       if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile):
         # nothing changed
@@ -127,7 +122,11 @@ proc commandCompileToC(graph: ModuleGraph) =
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
 
-  compileProject(graph)
+  if conf.symbolFiles == disabledSf:
+    setPipeLinePass(graph, CgenPass)
+  else:
+    setPipeLinePass(graph, SemPass)
+  compilePipelineProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
   if conf.symbolFiles == disabledSf:
@@ -160,33 +159,27 @@ proc commandCompileToJS(graph: ModuleGraph) =
     conf.exc = excCpp
     setTarget(conf.target, osJS, cpuJS)
     defineSymbol(conf.symbols, "ecmascript") # For backward compatibility
-    semanticPasses(graph)
+    setPipeLinePass(graph, JSgenPass)
-    registerPass(graph, JSgenPass)
+    compilePipelineProject(graph)
-    compileProject(graph)
     if optGenScript in conf.globalOptions:
       writeDepsFile(graph)
 
-proc interactivePasses(graph: ModuleGraph) =
+proc commandInteractive(graph: ModuleGraph) =
+  graph.config.setErrorMaxHighMaybe
   initDefines(graph.config.symbols)
   defineSymbol(graph.config.symbols, "nimscript")
   # note: seems redundant with -d:nimHasLibFFI
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
+  setPipeLinePass(graph, InterpreterPass)
-  registerPass(graph, semPass)
+  compilePipelineSystemModule(graph)
-  registerPass(graph, evalPass)
-
-proc commandInteractive(graph: ModuleGraph) =
-  graph.config.setErrorMaxHighMaybe
-  interactivePasses(graph)
-  compileSystemModule(graph)
   if graph.config.commandArgs.len > 0:
-    discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
+    discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {})
   else:
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
     var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
     let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
-    processModule(graph, m, idgen, s)
+    discard processPipelineModule(graph, m, idgen, s)
 
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
@@ -241,8 +234,6 @@ proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
 
-  # In "nim serve" scenario, each command must reset the registered passes
-  clearPasses(graph)
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
 

compiler/modulegraphs.nim

@@ -57,6 +57,18 @@ type
     sym*: PSym
     info*: TLineInfo
 
+  PipelinePass* = enum
+    NonePass
+    SemPass
+    JSgenPass
+    CgenPass
+    EvalPass
+    InterpreterPass
+    GenDependPass
+    Docgen2TexPass
+    Docgen2JsonPass
+    Docgen2Pass
+
   ModuleGraph* {.acyclic.} = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
@@ -104,6 +116,7 @@ type
     cacheCounters*: Table[string, BiggestInt] # IC: implemented
     cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
     passes*: seq[TPass]
+    pipelinePass*: PipelinePass
     onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}

compiler/modules.nim

@@ -11,13 +11,8 @@
 
 import
   ast, magicsys, msgs, options,
-  idents, lexer, passes, syntaxes, llstream, modulegraphs,
+  idents, lexer, syntaxes, modulegraphs,
-  lineinfos, pathutils, tables, packages
+  lineinfos, pathutils
-
-when defined(nimPreviewSlimSystem):
-  import std/[syncio, assertions]
-
-import ic / replayer
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
@@ -25,12 +20,12 @@ proc resetSystemArtifacts*(g: ModuleGraph) =
 template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
   getIdent(graph.cache, splitFile(filename).name)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
-proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
@@ -43,99 +38,16 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   partialInitModule(result, graph, fileIdx, filename)
   graph.registerModule(result)
 
-proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
-  var flags = flags
-  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
-  result = graph.getModule(fileIdx)
-
-  template processModuleAux(moduleStatus) =
-    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
-    var s: PLLStream
-    if sfMainModule in flags:
-      if graph.config.projectIsStdin: s = stdin.llStreamOpen
-      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
-    discard processModule(graph, result, idGeneratorFromModule(result), s)
-  if result == nil:
-    var cachedModules: seq[FileIndex]
-    result = moduleFromRodFile(graph, fileIdx, cachedModules)
-    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    if result == nil:
-      result = newModule(graph, fileIdx)
-      result.flags.incl flags
-      registerModule(graph, result)
-      processModuleAux("import")
-    else:
-      if sfSystemModule in flags:
-        graph.systemModule = result
-      partialInitModule(result, graph, fileIdx, filename)
-    for m in cachedModules:
-      registerModuleById(graph, m)
-      replayStateChanges(graph.packed[m.int].module, graph)
-      replayGenericCacheInformation(graph, m.int)
-  elif graph.isDirty(result):
-    result.flags.excl sfDirty
-    # reset module fields:
-    initStrTables(graph, result)
-    result.ast = nil
-    processModuleAux("import(dirty)")
-    graph.markClientsDirty(fileIdx)
-
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
-  # this is called by the semantic checking phase
-  assert graph.config != nil
-  result = compileModule(graph, fileIdx, {}, s)
-  graph.addDep(s, fileIdx)
-  # keep track of import relationships
-  if graph.config.hcrOn:
-    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
-  #if sfSystemModule in result.flags:
-  #  localError(result.info, errAttemptToRedefine, result.name.s)
-  # restore the notes for outer module:
-  graph.config.notes =
-    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
-    else: graph.config.foreignPackageNotes
-
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
 
-proc connectCallbacks*(graph: ModuleGraph) =
-  graph.includeFileCallback = includeModule
-  graph.importModuleCallback = importModule
-
-proc compileSystemModule*(graph: ModuleGraph) =
-  if graph.systemModule == nil:
-    connectCallbacks(graph)
-    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
-        graph.config.libpath / RelativeFile"system.nim")
-    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})
-
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
     fatal(conf, gCmdLineInfo, "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
-proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
-  connectCallbacks(graph)
-  let conf = graph.config
-  wantMainModule(conf)
-  configComplete(graph)
-
-  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
-  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
-  conf.projectMainIdx2 = projectFile
-
-  let packSym = getPackage(graph, projectFile)
-  graph.config.mainPackageId = packSym.getPackageId
-  graph.importStack.add projectFile
-
-  if projectFile == systemFileIdx:
-    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
-  else:
-    graph.compileSystemModule()
-    discard graph.compileModule(projectFile, {sfMainModule})
-
 proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym =
   result = graph.newModule(fileInfoIdx(graph.config, filename))
   registerModule(graph, result)

compiler/nimeval.nim

@@ -9,10 +9,16 @@
 
 ## exposes the Nim VM to clients.
 import
-  ast, astalgo, modules, passes, condsyms,
+  ast, modules, condsyms,
-  options, sem, llstream, lineinfos, vm,
+  options, llstream, lineinfos, vm,
   vmdef, modulegraphs, idents, os, pathutils,
-  passaux, scriptconfig, std/compilesettings
+  scriptconfig, std/compilesettings
+
+import pipelines
+
+
+when defined(nimPreviewSlimSystem):
+  import std/[assertions, syncio]
 
 type
   Interpreter* = ref object ## Use Nim as an interpreter with this object
@@ -76,7 +82,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) =
 
   let s = if scriptStream != nil: scriptStream
           else: llStreamOpen(findFile(i.graph.config, i.scriptName), fmRead)
-  processModule(i.graph, i.mainModule, i.idgen, s)
+  discard processPipelineModule(i.graph, i.mainModule, i.idgen, s)
 
 proc findNimStdLib*(): string =
   ## Tries to find a path to a valid "system.nim" file.
@@ -109,12 +115,10 @@ proc createInterpreter*(scriptName: string;
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
-  connectCallbacks(graph)
+  connectPipelineCallbacks(graph)
   initDefines(conf.symbols)
   for define in defines:
     defineSymbol(conf.symbols, define[0], define[1])
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
 
   for p in searchPaths:
     conf.searchPaths.add(AbsoluteDir p)
@@ -129,7 +133,8 @@ proc createInterpreter*(scriptName: string;
   if registerOps:
     vm.registerAdditionalOps() # Required to register parts of stdlib modules
   graph.vm = vm
-  graph.compileSystemModule()
+  setPipeLinePass(graph, EvalPass)
+  graph.compilePipelineSystemModule()
   result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName, idgen: idgen)
 
 proc destroyInterpreter*(i: Interpreter) =
@@ -159,13 +164,11 @@ proc runRepl*(r: TLLRepl;
   defineSymbol(conf.symbols, "nimscript")
   if supportNimscript: defineSymbol(conf.symbols, "nimconfig")
   when hasFFI: defineSymbol(graph.config.symbols, "nimffi")
-  registerPass(graph, verbosePass)
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
   var m = graph.makeStdinModule()
   incl(m.flags, sfMainModule)
   var idgen = idGeneratorFromModule(m)
 
   if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph, idgen)
-  graph.compileSystemModule()
+  setPipeLinePass(graph, InterpreterPass)
-  processModule(graph, m, idgen, llStreamOpenStdIn(r))
+  graph.compilePipelineSystemModule()
+  discard processPipelineModule(graph, m, idgen, llStreamOpenStdIn(r))

compiler/passes.nim

@@ -14,13 +14,22 @@ import
   options, ast, llstream, msgs,
   idents,
   syntaxes, modulegraphs, reorder,
-  lineinfos, pathutils, packages
+  lineinfos,
+  pipelineutils,
+  modules, pathutils, packages,
+  sem, semdata
+
+import ic/replayer
+
+export skipCodegen, resolveMod, prepareConfigNotes
 
 when defined(nimsuggest):
   import std/sha1
 
 when defined(nimPreviewSlimSystem):
-  import std/syncio
+  import std/[syncio, assertions]
+
+import std/tables
 
 type
   TPassData* = tuple[input: PNode, closeOutput: PNode]
@@ -38,12 +47,6 @@ proc makePass*(open: TPassOpen = nil,
   result.process = process
   result.isFrontend = isFrontend
 
-proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
-  # can be used by codegen passes to determine whether they should do
-  # something with `n`. Currently, this ignores `n` and uses the global
-  # error count instead.
-  result = config.errorCounter > 0
-
 const
   maxPasses = 10
 
@@ -80,13 +83,6 @@ proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray)
       if isNil(m): return false
   result = true
 
-proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex =
-  let fullPath = findModule(conf, module, relativeTo)
-  if fullPath.isEmpty:
-    result = InvalidFileIdx
-  else:
-    result = fileInfoIdx(conf, fullPath)
-
 proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
                       a: var TPassContextArray; m: PSym) =
   # XXX fixme this should actually be relative to the config file!
@@ -100,23 +96,6 @@ proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNod
       importStmt.add str
       if not processTopLevelStmt(graph, importStmt, a): break
 
-const
-  imperativeCode = {low(TNodeKind)..high(TNodeKind)} - {nkTemplateDef, nkProcDef, nkMethodDef,
-    nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma,
-    nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
-
-proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
-  # don't be verbose unless the module belongs to the main package:
-  if graph.config.belongsToProjectPackage(module):
-    graph.config.notes = graph.config.mainPackageNotes
-  else:
-    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
-    graph.config.notes = graph.config.foreignPackageNotes
-
-proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
-  result = true
-  #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
-
 proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                     stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
@@ -153,12 +132,11 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
         processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module
 
     checkFirstLineIndentation(p)
-    while true:
+    block processCode:
-      if graph.stopCompile(): break
+      if graph.stopCompile(): break processCode
-      var n = parseTopLevelStmt(p) # todo merge it
+      var n = parseTopLevelStmt(p)
-      if n.kind == nkEmpty: break
+      if n.kind == nkEmpty: break processCode
 
-      if true:
       # read everything, no streaming possible
       var sl = newNodeI(nkStmtList, n.info)
       sl.add n
@@ -169,26 +147,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
       if sfReorder in module.flags or codeReordering in graph.config.features:
         sl = reorder(graph, sl, module)
       discard processTopLevelStmt(graph, sl, a)
-        break
+
-      elif n.kind in imperativeCode:
-        # read everything until the next proc declaration etc.
-        var sl = newNodeI(nkStmtList, n.info)
-        sl.add n
-        var rest: PNode = nil
-        while true:
-          var n = parseTopLevelStmt(p)
-          if n.kind == nkEmpty or n.kind notin imperativeCode:
-            rest = n
-            break
-          sl.add n
-        #echo "-----\n", sl
-        if not processTopLevelStmt(graph, sl, a): break
-        if rest != nil:
-          #echo "-----\n", rest
-          if not processTopLevelStmt(graph, rest, a): break
-      else:
-        #echo "----- single\n", n
-        if not processTopLevelStmt(graph, n, a): break
     closeParser(p)
     if s.kind != llsStdIn: break
   closePasses(graph, a)
@@ -198,3 +157,99 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
     # They are responsible for closing the rod files. See `cbackend.nim`.
     closeRodFile(graph, module)
   result = true
+
+proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
+  result = graph.getModule(fileIdx)
+
+  template processModuleAux(moduleStatus) =
+    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
+    var s: PLLStream
+    if sfMainModule in flags:
+      if graph.config.projectIsStdin: s = stdin.llStreamOpen
+      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
+    discard processModule(graph, result, idGeneratorFromModule(result), s)
+  if result == nil:
+    var cachedModules: seq[FileIndex]
+    result = moduleFromRodFile(graph, fileIdx, cachedModules)
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    if result == nil:
+      result = newModule(graph, fileIdx)
+      result.flags.incl flags
+      registerModule(graph, result)
+      processModuleAux("import")
+    else:
+      if sfSystemModule in flags:
+        graph.systemModule = result
+      partialInitModule(result, graph, fileIdx, filename)
+    for m in cachedModules:
+      registerModuleById(graph, m)
+      replayStateChanges(graph.packed[m.int].module, graph)
+      replayGenericCacheInformation(graph, m.int)
+  elif graph.isDirty(result):
+    result.flags.excl sfDirty
+    # reset module fields:
+    initStrTables(graph, result)
+    result.ast = nil
+    processModuleAux("import(dirty)")
+    graph.markClientsDirty(fileIdx)
+
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
+  # this is called by the semantic checking phase
+  assert graph.config != nil
+  result = compileModule(graph, fileIdx, {}, s)
+  graph.addDep(s, fileIdx)
+  # keep track of import relationships
+  if graph.config.hcrOn:
+    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
+  #if sfSystemModule in result.flags:
+  #  localError(result.info, errAttemptToRedefine, result.name.s)
+  # restore the notes for outer module:
+  graph.config.notes =
+    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    else: graph.config.foreignPackageNotes
+
+proc connectCallbacks*(graph: ModuleGraph) =
+  graph.includeFileCallback = modules.includeModule
+  graph.importModuleCallback = importModule
+
+proc compileSystemModule*(graph: ModuleGraph) =
+  if graph.systemModule == nil:
+    connectCallbacks(graph)
+    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
+        graph.config.libpath / RelativeFile"system.nim")
+    discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule})
+
+proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
+  connectCallbacks(graph)
+  let conf = graph.config
+  wantMainModule(conf)
+  configComplete(graph)
+
+  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getPackageId
+  graph.importStack.add projectFile
+
+  if projectFile == systemFileIdx:
+    discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    graph.compileSystemModule()
+    discard graph.compileModule(projectFile, {sfMainModule})
+
+proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+  result = preparePContext(graph, module, idgen)
+
+proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
+  var c = PContext(context)
+  closePContext(graph, c, n)
+
+proc mySemProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
+  result = semWithPContext(PContext(context), n)
+
+const semPass* = makePass(mySemOpen, mySemProcess, mySemClose,
+                          isFrontend = true)

compiler/pipelines.nim

@@ -0,0 +1,254 @@
+import sem, cgen, modulegraphs, ast, llstream, parser, msgs,
+       lineinfos, reorder, options, semdata, cgendata, modules, pathutils,
+       packages, syntaxes, depends, vm
+
+import pipelineutils
+
+when not defined(leanCompiler):
+  import jsgen, docgen2
+
+import std/[syncio, objectdollar, assertions, tables]
+import renderer
+import ic/replayer
+
+
+proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) =
+  graph.pipelinePass = pass
+
+proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext): PNode =
+  case graph.pipelinePass
+  of CgenPass:
+    result = semNode
+    if bModule != nil:
+      genTopLevelStmt(BModule(bModule), result)
+  of JSgenPass:
+    when not defined(leanCompiler):
+      result = processJSCodeGen(bModule, semNode)
+  of GenDependPass:
+    result = addDotDependency(bModule, semNode)
+  of SemPass:
+    result = graph.emptyNode
+  of Docgen2Pass, Docgen2TexPass:
+    when not defined(leanCompiler):
+      result = processNode(bModule, semNode)
+  of Docgen2JsonPass:
+    when not defined(leanCompiler):
+      result = processNodeJson(bModule, semNode)
+  of EvalPass, InterpreterPass:
+    result = interpreterCode(bModule, semNode)
+  of NonePass:
+    doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+
+proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind,
+                      m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator,
+                      ) =
+  # XXX fixme this should actually be relative to the config file!
+  let relativeTo = toFullPath(graph.config, m.info)
+  for module in items(implicits):
+    # implicit imports should not lead to a module importing itself
+    if m.position != resolveMod(graph.config, module, relativeTo).int32:
+      var importStmt = newNodeI(nodeKind, m.info)
+      var str = newStrNode(nkStrLit, module)
+      str.info = m.info
+      importStmt.add str
+      message(graph.config, importStmt.info, hintProcessingStmt, $idgen[])
+      let semNode = semWithPContext(ctx, importStmt)
+      if semNode == nil or processPipeline(graph, semNode, bModule) == nil:
+        break
+
+proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
+                    stream: PLLStream): bool =
+  if graph.stopCompile(): return true
+  var
+    p: Parser
+    s: PLLStream
+    fileIdx = module.fileIdx
+
+  prepareConfigNotes(graph, module)
+  let ctx = preparePContext(graph, module, idgen)
+  let bModule: PPassContext =
+    case graph.pipelinePass
+    of CgenPass:
+      setupCgen(graph, module, idgen)
+    of JSgenPass:
+      when not defined(leanCompiler):
+        setupJSgen(graph, module, idgen)
+      else:
+        nil
+    of EvalPass, InterpreterPass:
+      setupEvalGen(graph, module, idgen)
+    of GenDependPass:
+      setupDependPass(graph, module, idgen)
+    of Docgen2Pass:
+      when not defined(leanCompiler):
+        openHtml(graph, module, idgen)
+      else:
+        nil
+    of Docgen2TexPass:
+      when not defined(leanCompiler):
+        openTex(graph, module, idgen)
+      else:
+        nil
+    of Docgen2JsonPass:
+      when not defined(leanCompiler):
+        openJson(graph, module, idgen)
+      else:
+        nil
+    of SemPass:
+      nil
+    of NonePass:
+      doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+      nil
+
+  if stream == nil:
+    let filename = toFullPathConsiderDirty(graph.config, fileIdx)
+    s = llStreamOpen(filename, fmRead)
+    if s == nil:
+      rawMessage(graph.config, errCannotOpenFile, filename.string)
+      return false
+  else:
+    s = stream
+
+  while true:
+    syntaxes.openParser(p, fileIdx, s, graph.cache, graph.config)
+
+    if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"):
+      # XXX what about caching? no processing then? what if I change the
+      # modules to include between compilation runs? we'd need to track that
+      # in ROD files. I think we should enable this feature only
+      # for the interactive mode.
+      if module.name.s != "nimscriptapi":
+        processImplicitImports graph, graph.config.implicitImports, nkImportStmt, module, ctx, bModule, idgen
+        processImplicitImports graph, graph.config.implicitIncludes, nkIncludeStmt, module, ctx, bModule, idgen
+
+    checkFirstLineIndentation(p)
+    block processCode:
+      if graph.stopCompile(): break processCode
+      var n = parseTopLevelStmt(p)
+      if n.kind == nkEmpty: break processCode
+      # read everything, no streaming possible
+      var sl = newNodeI(nkStmtList, n.info)
+      sl.add n
+      while true:
+        var n = parseTopLevelStmt(p)
+        if n.kind == nkEmpty: break
+        sl.add n
+      if sfReorder in module.flags or codeReordering in graph.config.features:
+        sl = reorder(graph, sl, module)
+      if graph.pipelinePass != EvalPass:
+        message(graph.config, sl.info, hintProcessingStmt, $idgen[])
+      var semNode = semWithPContext(ctx, sl)
+      discard processPipeline(graph, semNode, bModule)
+
+    closeParser(p)
+    if s.kind != llsStdIn: break
+  let finalNode = closePContext(graph, ctx, nil)
+  case graph.pipelinePass
+  of CgenPass:
+    if bModule != nil:
+      finalCodegenActions(graph, BModule(bModule), finalNode)
+  of JSgenPass:
+    when not defined(leanCompiler):
+      discard finalJSCodeGen(graph, bModule, finalNode)
+  of EvalPass, InterpreterPass:
+    discard interpreterCode(bModule, finalNode)
+  of SemPass, GenDependPass:
+    discard
+  of Docgen2Pass, Docgen2TexPass:
+    when not defined(leanCompiler):
+      discard closeDoc(graph, bModule, finalNode)
+  of Docgen2JsonPass:
+    when not defined(leanCompiler):
+      discard closeJson(graph, bModule, finalNode)
+  of NonePass:
+    doAssert false, "use setPipeLinePass to set a proper PipelinePass"
+
+  if graph.config.backend notin {backendC, backendCpp, backendObjc}:
+    # We only write rod files here if no C-like backend is active.
+    # The C-like backends have been patched to support the IC mechanism.
+    # They are responsible for closing the rod files. See `cbackend.nim`.
+    closeRodFile(graph, module)
+  result = true
+
+proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
+  result = graph.getModule(fileIdx)
+
+  template processModuleAux(moduleStatus) =
+    onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
+    var s: PLLStream
+    if sfMainModule in flags:
+      if graph.config.projectIsStdin: s = stdin.llStreamOpen
+      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
+    discard processPipelineModule(graph, result, idGeneratorFromModule(result), s)
+  if result == nil:
+    var cachedModules: seq[FileIndex]
+    result = moduleFromRodFile(graph, fileIdx, cachedModules)
+    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    if result == nil:
+      result = newModule(graph, fileIdx)
+      result.flags.incl flags
+      registerModule(graph, result)
+      processModuleAux("import")
+    else:
+      if sfSystemModule in flags:
+        graph.systemModule = result
+      partialInitModule(result, graph, fileIdx, filename)
+    for m in cachedModules:
+      registerModuleById(graph, m)
+      replayStateChanges(graph.packed[m.int].module, graph)
+      replayGenericCacheInformation(graph, m.int)
+  elif graph.isDirty(result):
+    result.flags.excl sfDirty
+    # reset module fields:
+    initStrTables(graph, result)
+    result.ast = nil
+    processModuleAux("import(dirty)")
+    graph.markClientsDirty(fileIdx)
+
+proc importPipelineModule(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
+  # this is called by the semantic checking phase
+  assert graph.config != nil
+  result = compilePipelineModule(graph, fileIdx, {}, s)
+  graph.addDep(s, fileIdx)
+  # keep track of import relationships
+  if graph.config.hcrOn:
+    graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx)
+  #if sfSystemModule in result.flags:
+  #  localError(result.info, errAttemptToRedefine, result.name.s)
+  # restore the notes for outer module:
+  graph.config.notes =
+    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    else: graph.config.foreignPackageNotes
+
+proc connectPipelineCallbacks*(graph: ModuleGraph) =
+  graph.includeFileCallback = modules.includeModule
+  graph.importModuleCallback = importPipelineModule
+
+proc compilePipelineSystemModule*(graph: ModuleGraph) =
+  if graph.systemModule == nil:
+    connectPipelineCallbacks(graph)
+    graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
+        graph.config.libpath / RelativeFile"system.nim")
+    discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule})
+
+proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
+  connectPipelineCallbacks(graph)
+  let conf = graph.config
+  wantMainModule(conf)
+  configComplete(graph)
+
+  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
+  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getPackageId
+  graph.importStack.add projectFile
+
+  if projectFile == systemFileIdx:
+    discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule})
+  else:
+    graph.compilePipelineSystemModule()
+    discard graph.compilePipelineModule(projectFile, {sfMainModule})

compiler/pipelineutils.nim

@@ -0,0 +1,26 @@
+import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages
+
+proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
+  # can be used by codegen passes to determine whether they should do
+  # something with `n`. Currently, this ignores `n` and uses the global
+  # error count instead.
+  result = config.errorCounter > 0
+
+proc resolveMod*(conf: ConfigRef; module, relativeTo: string): FileIndex =
+  let fullPath = findModule(conf, module, relativeTo)
+  if fullPath.isEmpty:
+    result = InvalidFileIdx
+  else:
+    result = fileInfoIdx(conf, fullPath)
+
+proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) =
+  # don't be verbose unless the module belongs to the main package:
+  if graph.config.belongsToProjectPackage(module):
+    graph.config.notes = graph.config.mainPackageNotes
+  else:
+    if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
+    graph.config.notes = graph.config.foreignPackageNotes
+
+proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
+  result = true
+  #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

compiler/scriptconfig.nim

@@ -11,10 +11,10 @@
 ## language.
 
 import
-  ast, modules, idents, passes, condsyms,
+  ast, modules, idents, condsyms,
-  options, sem, llstream, vm, vmdef, commands,
+  options, llstream, vm, vmdef, commands,
   os, times, osproc, wordrecg, strtabs, modulegraphs,
-  pathutils
+  pathutils, pipelines
 
 when defined(nimPreviewSlimSystem):
   import std/syncio
@@ -197,13 +197,11 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   conf.symbolFiles = disabledSf
 
   let graph = newModuleGraph(cache, conf)
-  connectCallbacks(graph)
+  connectPipelineCallbacks(graph)
   if freshDefines: initDefines(conf.symbols)
 
   defineSymbol(conf.symbols, "nimscript")
   defineSymbol(conf.symbols, "nimconfig")
-  registerPass(graph, semPass)
-  registerPass(graph, evalPass)
 
   conf.searchPaths.add(conf.libpath)
 
@@ -218,8 +216,9 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
   var vm = setupVM(m, cache, scriptName.string, graph, idgen)
   graph.vm = vm
 
-  graph.compileSystemModule()
+  graph.setPipeLinePass(EvalPass)
-  discard graph.processModule(m, vm.idgen, stream)
+  graph.compilePipelineSystemModule()
+  discard graph.processPipelineModule(m, vm.idgen, stream)
 
   # watch out, "newruntime" can be set within NimScript itself and then we need
   # to remember this:

compiler/sem.nim

@@ -13,7 +13,7 @@ import
   ast, strutils, options, astalgo, trees,
   wordrecg, ropes, msgs, idents, renderer, types, platform, math,
   magicsys, nversion, nimsets, semfold, modulepaths, importer,
-  procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
+  procfind, lookups, pragmas, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
   lowerings, plugins/active, lineinfos, strtabs, int128,
@@ -666,38 +666,37 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
         n.add prc.ast
   c.lastGenericIdx = c.generics.len
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext {.nosinks.} =
-  var c = newContext(graph, module)
+  result = newContext(graph, module)
-  c.idgen = idgen
+  result.idgen = idgen
-  c.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil)
+  result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil)
-  c.voidType = newType(tyVoid, nextTypeId(idgen), nil)
+  result.voidType = newType(tyVoid, nextTypeId(idgen), nil)
 
-  if c.p != nil: internalError(graph.config, module.info, "sem.myOpen")
+  if result.p != nil: internalError(graph.config, module.info, "sem.preparePContext")
-  c.semConstExpr = semConstExpr
+  result.semConstExpr = semConstExpr
-  c.semExpr = semExpr
+  result.semExpr = semExpr
-  c.semTryExpr = tryExpr
+  result.semTryExpr = tryExpr
-  c.semTryConstExpr = tryConstExpr
+  result.semTryConstExpr = tryConstExpr
-  c.computeRequiresInit = computeRequiresInit
+  result.computeRequiresInit = computeRequiresInit
-  c.semOperand = semOperand
+  result.semOperand = semOperand
-  c.semConstBoolExpr = semConstBoolExpr
+  result.semConstBoolExpr = semConstBoolExpr
-  c.semOverloadedCall = semOverloadedCall
+  result.semOverloadedCall = semOverloadedCall
-  c.semInferredLambda = semInferredLambda
+  result.semInferredLambda = semInferredLambda
-  c.semGenerateInstance = generateInstance
+  result.semGenerateInstance = generateInstance
-  c.semTypeNode = semTypeNode
+  result.semTypeNode = semTypeNode
-  c.instTypeBoundOp = sigmatch.instTypeBoundOp
+  result.instTypeBoundOp = sigmatch.instTypeBoundOp
-  c.hasUnresolvedArgs = hasUnresolvedArgs
+  result.hasUnresolvedArgs = hasUnresolvedArgs
-  c.templInstCounter = new int
+  result.templInstCounter = new int
 
-  pushProcCon(c, module)
+  pushProcCon(result, module)
-  pushOwner(c, c.module)
+  pushOwner(result, result.module)
 
-  c.moduleScope = openScope(c)
+  result.moduleScope = openScope(result)
-  c.moduleScope.addSym(module) # a module knows itself
+  result.moduleScope.addSym(module) # a module knows itself
 
   if sfSystemModule in module.flags:
     graph.systemModule = module
-  c.topLevelScope = openScope(c)
+  result.topLevelScope = openScope(result)
-  result = c
 
 proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool =
   if g.systemModule == nil: return false
@@ -771,8 +770,7 @@ proc recoverContext(c: PContext) =
   while getCurrOwner(c).kind != skModule: popOwner(c)
   while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next
 
-proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
+proc semWithPContext*(c: PContext, n: PNode): PNode {.nosinks.} =
-  var c = PContext(context)
   # no need for an expensive 'try' if we stop after the first error anyway:
   if c.config.errorMax <= 1:
     result = semStmtAndGenerateGenerics(c, n)
@@ -793,13 +791,13 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
       #if c.config.cmd == cmdIdeTools: findSuggest(c, n)
   storeRodNode(c, result)
 
+
 proc reportUnusedModules(c: PContext) =
   for i in 0..high(c.unusedImports):
     if sfUsed notin c.unusedImports[i][0].flags:
       message(c.config, c.unusedImports[i][1], warnUnusedImportX, c.unusedImports[i][0].name.s)
 
-proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
+proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode =
-  var c = PContext(context)
   if c.config.cmd == cmdIdeTools and not c.suggestionsMade:
     suggestSentinel(c)
   closeScope(c)         # close module's scope
@@ -814,6 +812,3 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   popOwner(c)
   popProcCon(c)
   sealRodFile(c)
-
-const semPass* = makePass(myOpen, myProcess, myClose,
-                          isFrontend = true)

compiler/suggest.nim

@@ -36,7 +36,7 @@ import algorithm, sets, prefixmatches, parseutils, tables
 from wordrecg import wDeprecated, wError, wAddr, wYield
 
 when defined(nimsuggest):
-  import passes, tables, pathutils # importer
+  import tables, pathutils # importer
 
 const
   sep = '\t'

compiler/vm.nim

@@ -13,7 +13,7 @@
 import semmacrosanity
 import
   std/[strutils, tables, parseutils],
-  msgs, vmdef, vmgen, nimsets, types, passes,
+  msgs, vmdef, vmgen, nimsets, types,
   parser, vmdeps, idents, trees, renderer, options, transf,
   gorgeimpl, lineinfos, btrees, macrocacheimpl,
   modulegraphs, sighashes, int128, vmprofiler
@@ -2309,7 +2309,7 @@ proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) =
   else:
     refresh(PCtx graph.vm, module, idgen)
 
-proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
+proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} =
   #var c = newEvalContext(module, emRepl)
   #c.features = {allowCast, allowInfiniteLoops}
   #pushStackFrame(c, newStackFrame())
@@ -2318,7 +2318,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext
   setupGlobalCtx(module, graph, idgen)
   result = PCtx graph.vm
 
-proc myProcess(c: PPassContext, n: PNode): PNode =
+proc interpreterCode*(c: PPassContext, n: PNode): PNode =
   let c = PCtx(c)
   # don't eval errornous code:
   if c.oldErrorCount == c.config.errorCounter:
@@ -2328,11 +2328,6 @@ proc myProcess(c: PPassContext, n: PNode): PNode =
     result = n
   c.oldErrorCount = c.config.errorCounter
 
-proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode =
-  result = myProcess(c, n)
-
-const evalPass* = makePass(myOpen, myProcess, myClose)
-
 proc evalConstExprAux(module: PSym; idgen: IdGenerator;
                       g: ModuleGraph; prc: PSym, n: PNode,
                       mode: TEvalMode): PNode =

doc/refc.md

@@ -138,6 +138,7 @@ The garbage collector won't try to free them, you need to call their respective
 when you are done with them or they will leak.
 
 
+
 Heap dump
 =========
 

nimsuggest/nimsuggest.nim

@@ -23,7 +23,7 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp
 # Do NOT import suggest. It will lead to weird bugs with
 # suggestionResultHook, because suggest.nim is included by sigmatch.
 # So we import that one instead.
-import compiler / [options, commands, modules, sem,
+import compiler / [options, commands, modules,
   passes, passaux, msgs,
   sigmatch, ast,
   idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper,