CaaS in-memory caching

removed some redundant filepath params and variables and
switched to canonical paths in most places

compiler/ast.nim

@@ -824,6 +824,14 @@ proc linkTo*(s: PSym, t: PType): PSym {.discardable.} =
   s.typ = t
   result = s
 
+template fileIdx*(c: PSym): int32 =
+  # XXX: this should be used only on module symbols
+  c.position.int32
+
+template filename*(c: PSym): string =
+  # XXX: this should be used only on module symbols
+  c.position.int32.toFileName
+
 proc appendToModule*(m: PSym, n: PNode) =
   ## The compiler will use this internally to add nodes that will be
   ## appended to the module after the sem pass

compiler/cgen.nim

@@ -1013,17 +1013,20 @@ proc rawNewModule(module: PSym, filename: string): BModule =
   result.nimTypesName = getTempName()
   result.PreventStackTrace = sfSystemModule in module.flags
 
-proc newModule(module: PSym, filename: string): BModule = 
+proc rawNewModule(module: PSym): BModule =
-  result = rawNewModule(module, filename)
+  result = rawNewModule(module, module.filename)
+
+proc newModule(module: PSym): BModule =
+  result = rawNewModule(module)
   if gModules.len <= module.position: gModules.setLen(module.position + 1)
   gModules[module.position] = result
 
   if (optDeadCodeElim in gGlobalOptions): 
     if (sfDeadCodeElim in module.flags): 
-      InternalError("added pending module twice: " & filename)
+      InternalError("added pending module twice: " & module.filename)
   
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym): PPassContext = 
-  result = newModule(module, filename)
+  result = newModule(module)
   if optGenIndex in gGlobalOptions and generatedHeader == nil:
     let f = if headerFile.len > 0: headerFile else: gProjectFull
     generatedHeader = rawNewModule(module,
@@ -1053,9 +1056,8 @@ proc writeHeader(m: BModule) =
 proc getCFile(m: BModule): string =
   result = changeFileExt(completeCFilePath(m.cfilename), cExt)
 
-proc myOpenCached(module: PSym, filename: string, 
+proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = 
-                  rd: PRodReader): PPassContext = 
+  var m = newModule(module)
-  var m = newModule(module, filename)
   readMergeInfo(getCFile(m), m)
   result = m
 
@@ -1150,9 +1152,15 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     if generatedHeader != nil: finishModule(generatedHeader)
     while gForwardedProcsCounter > 0: 
       for i in countup(0, high(gModules)):
-        finishModule(gModules[i])
+        # some modules (like stdin) may exist only in memory
+        # they won't have a cgen BModule for them and we must
+        # skip them
+        if gModules[i] != nil:
+          finishModule(gModules[i])
     for i in countup(0, high(gModules)): 
-      writeModule(gModules[i], pending=true)
+      # see above
+      if gModules[i] != nil:
+        writeModule(gModules[i], pending=true)
     writeMapping(gMapping)
     if generatedHeader != nil: writeHeader(generatedHeader)
 

compiler/depends.nim

@@ -17,7 +17,6 @@ proc generateDot*(project: string)
 type 
   TGen = object of TPassContext
     module*: PSym
-    filename*: string
   PGen = ref TGen
 
 var gDotGraph: PRope # the generated DOT file; we need a global variable
@@ -47,11 +46,10 @@ proc generateDot(project: string) =
       toRope(changeFileExt(extractFileName(project), "")), gDotGraph]), 
             changeFileExt(project, "dot"))
 
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym): PPassContext =
   var g: PGen
   new(g)
   g.module = module
-  g.filename = filename
   result = g
 
 const gendependPass* = makePass(open = myOpen, process = addDotDependency)

compiler/docgen.nim

@@ -337,7 +337,7 @@ proc writeOutput*(d: PDoc, filename, outExt: string, useWarning = false) =
     writeRope(content, getOutFile(filename, outExt), useWarning)
 
 proc CommandDoc*() =
-  var ast = parseFile(addFileExt(gProjectFull, nimExt))
+  var ast = parseFile(gProjectMainIdx)
   if ast == nil: return 
   var d = newDocumentor(gProjectFull, options.gConfigVars)
   d.hasToc = true

compiler/docgen2.nim

@@ -17,14 +17,13 @@ type
   TGen = object of TPassContext
     doc: PDoc
     module: PSym
-    filename: string
   PGen = ref TGen
 
 proc close(p: PPassContext, n: PNode): PNode =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
   if gWholeProject or sfMainModule in g.module.flags:
-    writeOutput(g.doc, g.filename, HtmlExt, useWarning)
+    writeOutput(g.doc, g.module.filename, HtmlExt, useWarning)
     try:
       generateIndex(g.doc)
     except EIO:
@@ -35,12 +34,11 @@ proc processNode(c: PPassContext, n: PNode): PNode =
   var g = PGen(c)
   generateDoc(g.doc, n)
 
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym): PPassContext = 
   var g: PGen
   new(g)
   g.module = module
-  g.filename = filename
+  var d = newDocumentor(module.filename, options.gConfigVars)
-  var d = newDocumentor(filename, options.gConfigVars)
   d.hasToc = true
   g.doc = d
   result = g

compiler/ecmasgen.nim

@@ -1549,9 +1549,8 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) =
   
 var globals: PGlobals
 
-proc newModule(module: PSym, filename: string): BModule = 
+proc newModule(module: PSym): BModule = 
   new(result)
-  result.filename = filename
   result.module = module
   if globals == nil: globals = newGlobals()
   
@@ -1612,11 +1611,11 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     var outfile = changeFileExt(completeCFilePath(m.filename), "js")
     discard writeRopeIfNotEqual(con(genHeader(), code), outfile)
 
-proc myOpenCached(s: PSym, filename: string, rd: PRodReader): PPassContext = 
+proc myOpenCached(s: PSym, rd: PRodReader): PPassContext = 
   InternalError("symbol files are not possible with the Ecmas code generator")
   result = nil
 
-proc myOpen(s: PSym, filename: string): PPassContext = 
+proc myOpen(s: PSym): PPassContext = 
-  result = newModule(s, filename)
+  result = newModule(s)
 
 const ecmasgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)

compiler/evals.nim

@@ -61,8 +61,7 @@ proc newStackFrame*(): PStackFrame =
   initIdNodeTable(result.mapping)
   result.params = @[]
 
-proc newEvalContext*(module: PSym, filename: string, 
+proc newEvalContext*(module: PSym, mode: TEvalMode): PEvalContext =
-                     mode: TEvalMode): PEvalContext = 
   new(result)
   result.module = module
   result.mode = mode
@@ -1392,7 +1391,7 @@ proc eval*(c: PEvalContext, n: PNode): PNode =
       stackTrace(c, n, errCannotInterpretNodeX, renderTree(n))
 
 proc evalConstExprAux(module: PSym, e: PNode, mode: TEvalMode): PNode = 
-  var p = newEvalContext(module, "", mode)
+  var p = newEvalContext(module, mode)
   var s = newStackFrame()
   s.call = e
   pushStackFrame(p, s)
@@ -1432,8 +1431,8 @@ proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
   dec(evalTemplateCounter)
   c.callsite = nil
 
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym): PPassContext = 
-  var c = newEvalContext(module, filename, emRepl)
+  var c = newEvalContext(module, emRepl)
   pushStackFrame(c, newStackFrame())
   result = c
 

compiler/importer.nim

@@ -32,12 +32,15 @@ proc getModuleName*(n: PNode): string =
     internalError(n.info, "getModuleName")
     result = ""
 
-proc checkModuleName*(n: PNode): string =
+proc checkModuleName*(n: PNode): int32 =
   # This returns the full canonical path for a given module import
-  var modulename = n.getModuleName
+  let modulename = n.getModuleName
-  result = findModule(modulename)
+  let fullPath = findModule(modulename)
-  if result.len == 0:
+  if fullPath.len == 0:
     LocalError(n.info, errCannotOpenFile, modulename)
+    result = InvalidFileIDX
+  else:
+    result = fullPath.fileInfoIdx
 
 proc rawImportSymbol(c: PContext, s: PSym) = 
   # This does not handle stubs, because otherwise loading on demand would be
@@ -111,8 +114,8 @@ proc evalImport(c: PContext, n: PNode): PNode =
   result = n
   for i in countup(0, sonsLen(n) - 1): 
     var f = checkModuleName(n.sons[i])
-    if f.len > 0:
+    if f != InvalidFileIDX:
-      var m = gImportModule(f)
+      var m = gImportModule(c.module, f)
       if sfDeprecated in m.flags: 
         Message(n.sons[i].info, warnDeprecated, m.name.s) 
       # ``addDecl`` needs to be done before ``importAllSymbols``!
@@ -123,8 +126,8 @@ proc evalFrom(c: PContext, n: PNode): PNode =
   result = n
   checkMinSonsLen(n, 2)
   var f = checkModuleName(n.sons[0])
-  if f.len > 0:
+  if f != InvalidFileIDX:
-    var m = gImportModule(f)
+    var m = gImportModule(c.module, f)
     n.sons[0] = newSymNode(m)
     addDecl(c, m)               # add symbol to symbol table of module
     for i in countup(1, sonsLen(n) - 1): importSymbol(c, n.sons[i], m)

compiler/lexer.nim

@@ -126,7 +126,7 @@ proc pushInd*(L: var TLexer, indent: int)
 
 proc popInd*(L: var TLexer)
 proc isKeyword*(kind: TTokType): bool
-proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream)
+proc openLexer*(lex: var TLexer, fileidx: int32, inputstream: PLLStream)
 proc rawGetTok*(L: var TLexer, tok: var TToken)
   # reads in the next token into tok and skips it
 proc getColumn*(L: TLexer): int
@@ -135,6 +135,9 @@ proc closeLexer*(lex: var TLexer)
 proc PrintTok*(tok: TToken)
 proc tokToStr*(tok: TToken): string
 
+proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) =
+  OpenLexer(lex, filename.fileInfoIdx, inputStream)
+
 proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "")
 
 proc isKeyword(kind: TTokType): bool = 
@@ -211,10 +214,10 @@ proc fillToken(L: var TToken) =
   L.base = base10
   L.ident = dummyIdent
   
-proc openLexer(lex: var TLexer, filename: string, inputstream: PLLStream) = 
+proc openLexer(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) = 
   openBaseLexer(lex, inputstream)
   lex.indentStack = @[0]
-  lex.fileIdx = filename.fileInfoIdx
+  lex.fileIdx = fileIdx
   lex.indentAhead = - 1
   inc(lex.Linenumber, inputstream.lineOffset) 
 

compiler/main.nim

@@ -12,11 +12,11 @@
 
 import 
   llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs, 
-  os, lists, condsyms, rodread, rodwrite, ropes, trees, 
+  os, lists, condsyms, rodread, rodwrite, ropes, trees, times,
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, ecmasgen,
   platform, nimconf, importer, passaux, depends, evals, types, idgen,
-  tables, docgen2, service, magicsys, parser
+  tables, docgen2, service, magicsys, parser, crc
 
 const
   has_LLVM_Backend = false
@@ -28,60 +28,113 @@ proc MainCommand*()
 
 # ------------------ module handling -----------------------------------------
 
+type
+  TModuleInMemory = object
+    compiledAt: float
+    crc: int
+    deps: seq[int32] ## XXX: slurped files are not currently tracked
+    needsRecompile: bool
+
 var
-  compMods = initTable[string, PSym]() # all compiled modules
+  gCompiledModules: seq[PSym] = @[]
+  gMemCacheData: seq[TModuleInMemory] = @[]
+    ## XXX: we should implement recycling of file IDs
+    ## if the user keeps renaming modules, the file IDs will keep growing
 
-# This expects a normalized module path
+proc getModule(fileIdx: int32): PSym =
-proc registerModule(filename: string, module: PSym) =
+  if fileIdx >= 0 and fileIdx < gCompiledModules.len:
-  compMods[filename] = module
+    result = gCompiledModules[fileIdx]
+  else:
+    result = nil
 
-# This expects a normalized module path
+template compiledAt(x: PSym): expr =
-proc getModule(filename: string): PSym =
+  gMemCacheData[x.position].compiledAt
-  result = compMods[filename]
 
-var gModulesCount = 0
+template crc(x: PSym): expr =
-proc newModule(filename: string): PSym = 
+  gMemCacheData[x.position].crc
+
+template addDep(x: Psym, dep: int32) =
+  gMemCacheData[x.position].deps.add(dep)
+
+proc checkDepMem(fileIdx: int32): bool =
+  template markDirty =
+    gMemCacheData[fileIdx].needsRecompile = true
+    return true
+
+  if optForceFullMake in gGlobalOptions or
+     curCaasCmd != lastCaasCmd: markDirty
+  
+  let crc = crcFromFile(fileIdx.toFilename)
+  if crc != gMemCacheData[fileIdx].crc: markDirty
+
+  for dep in gMemCacheData[fileIdx].deps:
+    if checkDepMem(dep): markDirty
+
+  return false
+
+proc newModule(fileIdx: int32): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID. 
   new(result)
   result.id = - 1             # for better error checking
   result.kind = skModule
+  let filename = fileIdx.toFilename
   result.name = getIdent(splitFile(filename).name)
   if not isNimrodIdentifier(result.name.s):
     rawMessage(errInvalidModuleName, result.name.s)
   
   result.owner = result       # a module belongs to itself
-  result.info = newLineInfo(filename, 1, 1)
+  result.info = newLineInfo(fileIdx, 1, 1)
-  result.position = gModulesCount
+  result.position = fileIdx
-  inc gModulesCount
+  
+  growCache gMemCacheData, fileIdx
+  growCache gCompiledModules, fileIdx
+  gCompiledModules[result.position] = result
+  
   incl(result.flags, sfUsed)
   initStrTable(result.tab)
-  RegisterModule(filename, result)
   StrTableAdd(result.tab, result) # a module knows itself
   
-proc CompileModule(filename: string, flags: TSymFlags): PSym
+proc compileModule(fileIdx: int32, flags: TSymFlags): PSym =
-proc importModule(filename: string): PSym = 
+  result = getModule(fileIdx)
+  if result == nil:
+    result = newModule(fileIdx)
+    var rd = handleSymbolFile(result)
+    result.flags = result.flags + flags
+    if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}:
+      rd = handleSymbolFile(result)
+      if result.id < 0: 
+        InternalError("handleSymbolFile should have set the module\'s ID")
+        return
+    else:
+      result.id = getID()
+    processModule(result, nil, rd)
+    gMemCacheData[fileIdx].compiledAt = gLastCmdTime
+    gMemCacheData[fileIdx].needsRecompile = false
+  else:
+    InternalAssert optCaasEnabled in gGlobalOptions
+    if checkDepMem(fileIdx):
+      gCompiledModules[fileIdx] = nil
+      result = CompileModule(fileIdx, flags)
+    else:
+      result = gCompiledModules[fileIdx]
+
+proc compileModule(filename: string, flags: TSymFlags): PSym =
+  result = compileModule(filename.fileInfoIdx, flags)
+
+proc importModule(s: PSym, fileIdx: int32): PSym =
   # this is called by the semantic checking phase
-  result = getModule(filename)
+  result = getModule(fileIdx)
   if result == nil: 
     # compile the module
-    result = compileModule(filename, {})
+    result = compileModule(fileIdx, {})
+    if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
   elif sfSystemModule in result.flags: 
     LocalError(result.info, errAttemptToRedefine, result.Name.s)
 
-proc CompileModule(filename: string, flags: TSymFlags): PSym =
+proc includeModule(s: PSym, fileIdx: int32): PNode =
-  var rd: PRodReader = nil
+  result = syntaxes.parseFile(fileIdx)
-  var f = addFileExt(filename, nimExt)
+  if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
-  result = newModule(f)
-  result.flags = result.flags + flags
-  if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: 
-    rd = handleSymbolFile(result, f)
-    if result.id < 0: 
-      InternalError("handleSymbolFile should have set the module\'s ID")
-      return
-  else:
-    result.id = getID()
-  processModule(result, f, nil, rd)
 
 proc `==^`(a, b: string): bool =
   try:
@@ -91,7 +144,8 @@ proc `==^`(a, b: string): bool =
 
 proc compileSystemModule =
   if magicsys.SystemModule == nil:
-    discard CompileModule(options.libpath /"system", {sfSystemModule})
+    SystemFileIdx = fileInfoIdx(options.libpath/"system.nim")
+    discard CompileModule(SystemFileIdx, {sfSystemModule})
 
 proc CompileProject(projectFile = gProjectFull) =
   let systemFile = options.libpath / "system"
@@ -140,6 +194,8 @@ proc CommandCompileToC =
   compileProject()
   if gCmd != cmdRun:
     extccomp.CallCCompiler(changeFileExt(gProjectFull, ""))
+  # caas will keep track only of the compilation commands
+  lastCaasCmd = curCaasCmd
 
 when has_LLVM_Backend:
   proc CommandCompileToLLVM =
@@ -171,7 +227,7 @@ proc InteractivePasses =
 var stdinModule: PSym
 proc makeStdinModule: PSym =
   if stdinModule == nil:
-    stdinModule = newModule("stdin")
+    stdinModule = newModule(gCmdLineInfo.fileIndex)
     stdinModule.id = getID()
   result = stdinModule
 
@@ -184,24 +240,23 @@ proc CommandInteractive =
   else:
     var m = makeStdinModule()
     incl(m.flags, sfMainModule)
-    processModule(m, "stdin", LLStreamOpenStdIn(), nil)
+    processModule(m, LLStreamOpenStdIn(), nil)
 
 const evalPasses = [verbosePass, semPass, evalPass]
 
-proc evalNim(nodes: PNode, module: PSym, filename: string) =
+proc evalNim(nodes: PNode, module: PSym) =
-  # we don't want to mess with gPasses here, because in nimrod serve
+  carryPasses(nodes, module, evalPasses)
-  # scenario, it may be set up properly for normal cgenPass() compilation
-  carryPasses(nodes, module, filename, evalPasses)
 
 proc commandEval(exp: string) =
   if SystemModule == nil:
     InteractivePasses()
     compileSystemModule()
   var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(echoExp.parseString, makeStdinModule(), "stdin")
+  evalNim(echoExp.parseString, makeStdinModule())
 
 proc CommandPretty =
-  var module = parseFile(addFileExt(mainCommandArg(), NimExt))
+  var projectFile = addFileExt(mainCommandArg(), NimExt)
+  var module = parseFile(projectFile.fileInfoIdx)
   if module != nil: 
     renderModule(module, getOutFile(mainCommandArg(), "pretty." & NimExt))
   
@@ -231,16 +286,18 @@ proc CommandSuggest =
 proc wantMainModule =
   if gProjectFull.len == 0:
     Fatal(gCmdLineInfo, errCommandExpectsFilename)
+  gProjectMainIdx = addFileExt(gProjectFull, nimExt).fileInfoIdx
   
 proc MainCommand =
   # In "nimrod serve" scenario, each command must reset the registered passes
   clearPasses()
+  gLastCmdTime = epochTime()
   appendStr(searchPaths, options.libpath)
   if gProjectFull.len != 0:
     # current path is always looked first for modules
     prependStr(searchPaths, gProjectPath)
   setID(100)
-  passes.gIncludeFile = syntaxes.parseFile
+  passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   case command.normalize
   of "c", "cc", "compile", "compiletoc": 
@@ -323,7 +380,7 @@ proc MainCommand =
   of "parse": 
     gCmd = cmdParse
     wantMainModule()
-    discard parseFile(addFileExt(gProjectFull, nimExt))
+    discard parseFile(gProjectMainIdx)
   of "scan": 
     gCmd = cmdScan
     wantMainModule()
@@ -343,9 +400,14 @@ proc MainCommand =
       wantMainModule()
       CommandSuggest()
   of "serve":
-    gCmd = cmdIdeTools
+    gGlobalOptions.incl(optCaasEnabled)
     msgs.gErrorMax = high(int)  # do not stop after first error     
     serve(MainCommand)
     
   else: rawMessage(errInvalidCommandX, command)
   
+  if msgs.gErrorCounter == 0 and gCmd notin {cmdInterpret, cmdRun}:
+    rawMessage(hintSuccessX, [$gLinesCompiled,
+               formatFloat(epochTime() - gLastCmdTime, ffDecimal, 3),
+               formatSize(getTotalMem())])
+

compiler/msgs.nim

@@ -416,9 +416,13 @@ type
   ERecoverableError* = object of EInvalidValue
   ESuggestDone* = object of EBase
 
+const
+  InvalidFileIDX* = int32(-1)
+
 var
   filenameToIndexTbl = initTable[string, int32]()
-  fileInfos: seq[TFileInfo] = @[]
+  fileInfos*: seq[TFileInfo] = @[]
+  SystemFileIdx*: int32
 
 proc newFileInfo(fullPath, projPath: string): TFileInfo =
   result.fullPath = fullPath

compiler/nimrod.nim

@@ -14,7 +14,7 @@ when defined(gcc) and defined(windows):
     {.link: "icons/nimrod_icon.o".}
 
 import 
-  times, commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, 
+  commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, 
   extccomp, strutils, os, platform, main, parseopt, service
 
 when hasTinyCBackend:
@@ -32,7 +32,6 @@ proc prependCurDir(f: string): string =
     result = f
 
 proc HandleCmdLine() =
-  var start = epochTime()
   if paramCount() == 0:
     writeCommandLineUsage()
   else:
@@ -60,10 +59,6 @@ proc HandleCmdLine() =
       when hasTinyCBackend:
         if gCmd == cmdRun:
           tccgen.run()
-      if gCmd notin {cmdInterpret, cmdRun}:
-        rawMessage(hintSuccessX, [$gLinesCompiled,
-                   formatFloat(epochTime() - start, ffDecimal, 3),
-                   formatSize(getTotalMem())])
       if optRun in gGlobalOptions:
         if gCmd == cmdCompileToEcmaScript:
           var ex = quoteIfContainsWhite(

compiler/options.nim

@@ -43,6 +43,7 @@ type                          # please make sure we have under 32 options
     optGenMapping,            # generate a mapping file
     optRun,                   # run the compiled project
     optSymbolFiles,           # use symbol files for speeding up compilation
+    optCaasEnabled            # compiler-as-a-service is running
     optSkipConfigFile,        # skip the general config file
     optSkipProjConfigFile,    # skip the project's config file
     optSkipUserConfigFile,    # skip the users's config file
@@ -96,6 +97,7 @@ var
   gNumberOfProcessors*: int   # number of processors
   gWholeProject*: bool # for 'doc2': output any dependency
   gEvalExpr*: string          # expression for idetools --eval
+  gLastCmdTime*: float        # when caas is enabled, we measure each command
   
 const 
   genSubDir* = "nimcache"
@@ -115,6 +117,7 @@ var
   gProjectName* = "" # holds a name like 'nimrod'
   gProjectPath* = "" # holds a path like /home/alice/projects/nimrod/compiler/
   gProjectFull* = "" # projectPath/projectName
+  gProjectMainIdx*: int32 # the canonical path id of the main module
   nimcacheDir* = ""
   command* = "" # the main command (e.g. cc, check, scan, etc)
   commandArgs*: seq[string] = @[] # any arguments after the main command

compiler/parser.nim

@@ -64,11 +64,14 @@ proc parseCase(p: var TParser): PNode
 proc getTok(p: var TParser) = 
   rawGetTok(p.lex, p.tok)
 
-proc OpenParser(p: var TParser, filename: string, inputStream: PLLStream) = 
+proc OpenParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream) =
   initToken(p.tok)
-  OpenLexer(p.lex, filename, inputstream)
+  OpenLexer(p.lex, fileIdx, inputstream)
   getTok(p)                   # read the first token
 
+proc OpenParser*(p: var TParser, filename: string, inputStream: PLLStream) =
+  openParser(p, filename.fileInfoIdx, inputStream)
+
 proc CloseParser(p: var TParser) = 
   CloseLexer(p.lex)
 

compiler/passaux.nim

@@ -12,7 +12,7 @@
 import 
   strutils, ast, astalgo, passes, msgs, options, idgen
 
-proc verboseOpen(s: PSym, filename: string): PPassContext = 
+proc verboseOpen(s: PSym): PPassContext =
   #MessageOut('compiling ' + s.name.s);
   result = nil                # we don't need a context
   if gVerbosity > 0: rawMessage(hintProcessing, s.name.s)

compiler/passes.nim

@@ -21,9 +21,9 @@ type
    
   PPassContext* = ref TPassContext
 
-  TPassOpen* = proc (module: PSym, filename: string): PPassContext {.nimcall.}
+  TPassOpen* = proc (module: PSym): PPassContext {.nimcall.}
-  TPassOpenCached* = proc (module: PSym, filename: string,
+  TPassOpenCached* =
-                           rd: PRodReader): PPassContext {.nimcall.}
+    proc (module: PSym, rd: PRodReader): PPassContext {.nimcall.}
   TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.}
   TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
 
@@ -50,13 +50,12 @@ proc makePass*(open: TPassOpen = nil,
   # processed in a pipeline. The compiler never looks at a whole module
   # any longer. However, this is simple to change, as new passes may perform
   # whole program optimizations. For now, we avoid it to save a lot of memory.
-proc processModule*(module: PSym, filename: string, stream: PLLStream, 
+proc processModule*(module: PSym, stream: PLLStream, rd: PRodReader)
-                    rd: PRodReader)
 
 # the semantic checker needs these:
 var 
-  gImportModule*: proc (filename: string): PSym {.nimcall.}
+  gImportModule*: proc (m: PSym, fileIdx: int32): PSym {.nimcall.}
-  gIncludeFile*: proc (filename: string): PNode {.nimcall.}
+  gIncludeFile*: proc (m: PSym, fileIdx: int32): PNode {.nimcall.}
 
 # implementation
 
@@ -96,30 +95,28 @@ proc registerPass*(p: TPass) =
   gPasses[gPassesLen] = p
   inc(gPassesLen)
 
-proc carryPass*(p: TPass, module: PSym, filename: string,
+proc carryPass*(p: TPass, module: PSym, m: TPassData): TPassData =
-                m: TPassData): TPassData =
+  var c = p.open(module)
-  var c = p.open(module, filename)
   result.input = p.process(c, m.input)
   result.closeOutput = if p.close != nil: p.close(c, m.closeOutput)
                        else: m.closeOutput
 
-proc carryPasses*(nodes: PNode, module: PSym, file: string, passes: TPasses) =
+proc carryPasses*(nodes: PNode, module: PSym, passes: TPasses) =
   var passdata: TPassData
   passdata.input = nodes
   for pass in passes:
-    passdata = carryPass(pass, module, file, passdata)
+    passdata = carryPass(pass, module, passdata)
 
-proc openPasses(a: var TPassContextArray, module: PSym, filename: string) = 
+proc openPasses(a: var TPassContextArray, module: PSym) =
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].open): 
-      a[i] = gPasses[i].open(module, filename)
+      a[i] = gPasses[i].open(module)
     else: a[i] = nil
   
-proc openPassesCached(a: var TPassContextArray, module: PSym, filename: string, 
+proc openPassesCached(a: var TPassContextArray, module: PSym, rd: PRodReader) =
-                      rd: PRodReader) = 
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].openCached): 
-      a[i] = gPasses[i].openCached(module, filename, rd)
+      a[i] = gPasses[i].openCached(module, rd)
       if a[i] != nil: 
         a[i].fromCache = true
     else:
@@ -162,15 +159,16 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
     importStmt.addSon str
     if not processTopLevelStmt(importStmt, a): break
   
-proc processModule(module: PSym, filename: string, stream: PLLStream, 
+proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
-                   rd: PRodReader) = 
   var 
     p: TParsers
     a: TPassContextArray
     s: PLLStream
+    fileIdx = module.fileIdx
   if rd == nil: 
-    openPasses(a, module, filename)
+    openPasses(a, module)
     if stream == nil: 
+      let filename = fileIdx.toFilename
       s = LLStreamOpen(filename, fmRead)
       if s == nil: 
         rawMessage(errCannotOpenFile, filename)
@@ -178,7 +176,7 @@ proc processModule(module: PSym, filename: string, stream: PLLStream,
     else: 
       s = stream
     while true: 
-      openParsers(p, filename, s)
+      openParsers(p, fileIdx, s)
 
       if sfSystemModule notin module.flags:
         # XXX what about caching? no processing then? what if I change the 
@@ -199,7 +197,7 @@ proc processModule(module: PSym, filename: string, stream: PLLStream,
     # id synchronization point for more consistent code generation:
     IDsynchronizationPoint(1000)
   else:
-    openPassesCached(a, module, filename, rd)
+    openPassesCached(a, module, rd)
     var n = loadInitSection(rd)
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)

compiler/rodread.nim

@@ -126,7 +126,7 @@ type
     s: cstring               # mmap'ed file contents
     options: TOptions
     reason: TReasonForRecompile
-    modDeps: TStringSeq
+    modDeps: seq[int32]
     files: TStringSeq
     dataIdx: int             # offset of start of data section
     convertersIdx: int       # offset of start of converters section
@@ -145,7 +145,7 @@ type
 
 var rodCompilerprocs*: TStrTable
 
-proc handleSymbolFile*(module: PSym, filename: string): PRodReader
+proc handleSymbolFile*(module: PSym): PRodReader
 # global because this is needed by magicsys
 proc loadInitSection*(r: PRodReader): PNode
 
@@ -602,7 +602,7 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
     of "DEPS":
       inc(r.pos)              # skip ':'
       while r.s[r.pos] > '\x0A': 
-        r.modDeps.add r.files[decodeVInt(r.s, r.pos)]
+        r.modDeps.add int32(decodeVInt(r.s, r.pos))
         if r.s[r.pos] == ' ': inc(r.pos)
     of "INTERF": 
       r.interfIdx = r.pos + 2
@@ -699,10 +699,11 @@ type
     reason*: TReasonForRecompile
     rd*: PRodReader
     crc*: TCrc32
+    crcDone*: bool
 
   TFileModuleMap = seq[TFileModuleRec]
 
-var gMods: TFileModuleMap = @[]
+var gMods*: TFileModuleMap = @[]
 
 proc decodeSymSafePos(rd: PRodReader, offset: int, info: TLineInfo): PSym = 
   # all compiled modules
@@ -720,6 +721,10 @@ proc findSomeWhere(id: int) =
       if d != invalidKey:
         echo "found id ", id, " in ", gMods[i].filename
 
+proc getReader(moduleId: int): PRodReader =
+  InternalAssert moduleId >= 0 and moduleId < gMods.len
+  result = gMods[moduleId].rd
+
 proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym = 
   result = PSym(IdTableGet(r.syms, id))
   if result == nil: 
@@ -732,25 +737,15 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
         var x = ""
         encodeVInt(id, x)
         InternalError(info, "missing from both indexes: +" & x)
-      # find the reader with the correct moduleID:
+      var rd = getReader(moduleID)
-      for i in countup(0, high(gMods)): 
+      d = IITableGet(rd.index.tab, id)
-        var rd = gMods[i].rd
+      if d != invalidKey: 
-        if rd != nil: 
+        result = decodeSymSafePos(rd, d, info)
-          if rd.moduleID == moduleID: 
+      else:
-            d = IITableGet(rd.index.tab, id)
+        var x = ""
-            if d != invalidKey: 
+        encodeVInt(id, x)
-              result = decodeSymSafePos(rd, d, info)
+        when false: findSomeWhere(id)
-              break 
+        InternalError(info, "rrGetSym: no reader found: +" & x)
-            else:
-              var x = ""
-              encodeVInt(id, x)
-              when false: findSomeWhere(id)
-              InternalError(info, "rrGetSym: no reader found: +" & x)
-          else:
-            #if IiTableGet(rd.index.tab, id) <> invalidKey then
-            # XXX expensive check!
-            #InternalError(info,
-            #'id found in other module: +' + ropeToStr(encodeInt(id)))
     else: 
       # own symbol:
       result = decodeSymSafePos(r, d, info)
@@ -790,26 +785,31 @@ proc loadMethods(r: PRodReader) =
     r.methods.add(rrGetSym(r, d, UnknownLineInfo()))
     if r.s[r.pos] == ' ': inc(r.pos)
 
-proc getModuleIdx(filename: string): int = 
+proc GetCRC*(fileIdx: int32): TCrc32 =
-  for i in countup(0, high(gMods)): 
+  InternalAssert fileIdx >= 0 and fileIdx < gMods.len
-    if gMods[i].filename == filename: return i
-  result = len(gMods)
-  setlen(gMods, result + 1)
 
-proc checkDep(filename: string): TReasonForRecompile =
+  if gMods[fileIdx].crcDone:
-  assert(not isNil(filename)) 
+    return gMods[fileIdx].crc
-  var idx = getModuleIdx(filename)
+  
-  if gMods[idx].reason != rrEmpty: 
+  result = crcFromFile(fileIdx.toFilename)
+  gMods[fileIdx].crc = result
+
+template growCache*(cache, pos) =
+  if cache.len <= fileIdx: cache.setLen(pos+1)
+
+proc checkDep(fileIdx: int32): TReasonForRecompile =
+  assert fileIdx != InvalidFileIDX
+  growCache gMods, fileIdx
+  if gMods[fileIdx].reason != rrEmpty: 
     # reason has already been computed for this module:
-    return gMods[idx].reason
+    return gMods[fileIdx].reason
-  var crc: TCrc32 = crcFromFile(filename)
+  let filename = fileIdx.toFilename
-  gMods[idx].reason = rrNone  # we need to set it here to avoid cycles
+  var crc = GetCRC(fileIdx)
-  gMods[idx].filename = filename
+  gMods[fileIdx].reason = rrNone  # we need to set it here to avoid cycles
-  gMods[idx].crc = crc
   result = rrNone
   var r: PRodReader = nil
   var rodfile = toGeneratedFile(filename, RodExt)
-  r = newRodReader(rodfile, crc, idx)
+  r = newRodReader(rodfile, crc, fileIdx)
   if r == nil: 
     result = (if ExistsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
   else:
@@ -819,7 +819,7 @@ proc checkDep(filename: string): TReasonForRecompile =
       # NOTE: we need to process the entire module graph so that no ID will
       # be used twice! However, compilation speed does not suffer much from
       # this, since results are cached.
-      var res = checkDep(options.libpath / addFileExt("system", nimExt))
+      var res = checkDep(SystemFileIdx)
       if res != rrNone: result = rrModDeps
       for i in countup(0, high(r.modDeps)): 
         res = checkDep(r.modDeps[i])
@@ -832,19 +832,19 @@ proc checkDep(filename: string): TReasonForRecompile =
     # recompilation is necessary:
     if r != nil: memfiles.close(r.memFile)
     r = nil
-  gMods[idx].rd = r
+  gMods[fileIdx].rd = r
-  gMods[idx].reason = result  # now we know better
+  gMods[fileIdx].reason = result  # now we know better
   
-proc handleSymbolFile(module: PSym, filename: string): PRodReader = 
+proc handleSymbolFile(module: PSym): PRodReader = 
+  let fileIdx = module.fileIdx
   if optSymbolFiles notin gGlobalOptions: 
     module.id = getID()
     return nil
   idgen.loadMaxIds(options.gProjectPath / options.gProjectName)
 
-  discard checkDep(filename)
+  discard checkDep(fileIdx)
-  var idx = getModuleIdx(filename)
+  if gMods[fileIdx].reason == rrEmpty: InternalError("handleSymbolFile")
-  if gMods[idx].reason == rrEmpty: InternalError("handleSymbolFile")
+  result = gMods[fileIdx].rd
-  result = gMods[idx].rd
   if result != nil: 
     module.id = result.moduleID
     IdTablePut(result.syms, module, module)
@@ -855,14 +855,6 @@ proc handleSymbolFile(module: PSym, filename: string): PRodReader =
   else:
     module.id = getID()
 
-proc GetCRC*(filename: string): TCrc32 = 
-  for i in countup(0, high(gMods)): 
-    if gMods[i].filename == filename: return gMods[i].crc
-  
-  result = crcFromFile(filename)
-  #var idx = getModuleIdx(filename)
-  #result = gMods[idx].crc
-
 proc rawLoadStub(s: PSym) =
   if s.kind != skStub: InternalError("loadStub")
   var rd = gMods[s.position].rd

compiler/rodwrite.nim

@@ -31,14 +31,13 @@ type
     converters, methods: string
     init: string
     data: string
-    filename: string
     sstack: TSymSeq          # a stack of symbols to process
     tstack: TTypeSeq         # a stack of types to process
     files: TStringSeq
 
   PRodWriter = ref TRodWriter
 
-proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter
+proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter
 proc addModDep(w: PRodWriter, dep: string)
 proc addInclDep(w: PRodWriter, dep: string)
 proc addInterfaceSym(w: PRodWriter, s: PSym)
@@ -63,7 +62,10 @@ proc fileIdx(w: PRodWriter, filename: string): int =
   setlen(w.files, result + 1)
   w.files[result] = filename
 
-proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter = 
+template filename*(w: PRodWriter): string =
+  w.module.filename
+
+proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter = 
   new(result)
   result.sstack = @[]
   result.tstack = @[]
@@ -71,7 +73,6 @@ proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter =
   InitIITable(result.imports.tab)
   result.index.r = ""
   result.imports.r = ""
-  result.filename = modfilename
   result.crc = crc
   result.module = module
   result.defines = getDefines()
@@ -570,9 +571,9 @@ proc process(c: PPassContext, n: PNode): PNode =
   else: 
     nil
 
-proc myOpen(module: PSym, filename: string): PPassContext =
+proc myOpen(module: PSym): PPassContext =
   if module.id < 0: InternalError("rodwrite: module ID not set")
-  var w = newRodWriter(filename, rodread.GetCRC(module.info.toFullPath), module)
+  var w = newRodWriter(module.fileIdx.GetCRC, module)
   rawAddInterfaceSym(w, module)
   result = w
 

compiler/sem.nim

@@ -158,7 +158,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
     GlobalError(n.info, errRecursiveDependencyX, sym.name.s)
 
   if c.evalContext == nil:
-    c.evalContext = newEvalContext(c.module, "", emStatic)
+    c.evalContext = newEvalContext(c.module, emStatic)
     c.evalContext.getType = proc (n: PNode): PNode =
       var e = tryExpr(c, n)
       if e == nil:
@@ -201,8 +201,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
 proc semExprNoFlags(c: PContext, n: PNode): PNode {.procvar.} = 
   result = semExpr(c, n, {})
 
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym): PPassContext =
-  var c = newContext(module, filename)
+  var c = newContext(module)
   if c.p != nil: InternalError(module.info, "sem.myOpen")
   c.semConstExpr = semConstExpr
   c.semExpr = semExprNoFlags
@@ -222,9 +222,8 @@ proc myOpen(module: PSym, filename: string): PPassContext =
   openScope(c.tab)            # scope for the module's symbols  
   result = c
 
-proc myOpenCached(module: PSym, filename: string, 
+proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
-                  rd: PRodReader): PPassContext = 
+  result = myOpen(module)
-  result = myOpen(module, filename)
   for m in items(rd.methods): methodDef(m, true)
 
 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 

compiler/semdata.nim

@@ -81,7 +81,6 @@ type
                               filter: TSymKinds): PNode {.nimcall.}
     semTypeNode*: proc(c: PContext, n: PNode, prev: PType): PType {.nimcall.}
     includedFiles*: TIntSet    # used to detect recursive include files
-    filename*: string          # the module's filename
     userPragmas*: TStrTable
     evalContext*: PEvalContext
     UnknownIdents*: TIntSet    # ids of all unknown identifiers to prevent
@@ -90,12 +89,16 @@ type
 var
   gGenericsCache: PGenericsCache # save for modularity
 
+proc filename*(c: PContext): string =
+  # the module's filename
+  return c.module.filename
+
 proc newGenericsCache*(): PGenericsCache =
   new(result)
   initIdTable(result.InstTypes)
   result.generics = @[]
 
-proc newContext*(module: PSym, nimfile: string): PContext
+proc newContext*(module: PSym): PContext
 
 proc lastOptionEntry*(c: PContext): POptionEntry
 proc newOptionEntry*(): POptionEntry
@@ -152,7 +155,7 @@ proc newOptionEntry(): POptionEntry =
   result.dynlib = nil
   result.notes = gNotes
 
-proc newContext(module: PSym, nimfile: string): PContext = 
+proc newContext(module: PSym): PContext =
   new(result)
   InitSymTab(result.tab)
   result.AmbiguousSymbols = initIntset()
@@ -164,7 +167,6 @@ proc newContext(module: PSym, nimfile: string): PContext =
   result.threadEntries = @[]
   result.converters = @[]
   result.patterns = @[]
-  result.filename = nimfile
   result.includedFiles = initIntSet()
   initStrTable(result.userPragmas)
   if optSymbolFiles notin gGlobalOptions:

compiler/semstmts.nim

@@ -892,13 +892,12 @@ proc evalInclude(c: PContext, n: PNode): PNode =
   addSon(result, n)
   for i in countup(0, sonsLen(n) - 1): 
     var f = checkModuleName(n.sons[i])
-    if f.len > 0:
+    if f != InvalidFileIDX:
-      var fileIndex = f.fileInfoIdx
+      if ContainsOrIncl(c.includedFiles, f): 
-      if ContainsOrIncl(c.includedFiles, fileIndex): 
+        LocalError(n.info, errRecursiveDependencyX, f.toFilename)
-        LocalError(n.info, errRecursiveDependencyX, f.extractFilename)
       else:
-        addSon(result, semStmt(c, gIncludeFile(f)))
+        addSon(result, semStmt(c, gIncludeFile(c.module, f)))
-        Excl(c.includedFiles, fileIndex)
+        Excl(c.includedFiles, f)
   
 proc setLine(n: PNode, info: TLineInfo) =
   for i in 0 .. <safeLen(n): setLine(n.sons[i], info)

compiler/service.nim

@@ -19,8 +19,13 @@ import
 # repeated CRC calculations may turn out to be too slow.
 
 var
-  arguments*: string = ""     # the arguments to be passed to the program that
+  curCaasCmd* = ""
-                              # should be run
+  lastCaasCmd* = ""
+    # in caas mode, the list of defines and options will be given at start-up?
+    # it's enough to check that the previous compilation command is the same?
+  arguments* = ""
+    # the arguments to be passed to the program that
+    # should be run
 
 proc ProcessCmdLine*(pass: TCmdLinePass, cmd: string) =
   var p = parseopt.initOptParser(cmd)
@@ -56,14 +61,18 @@ proc ProcessCmdLine*(pass: TCmdLinePass, cmd: string) =
       rawMessage(errArgsNeedRunOption, [])
 
 proc serve*(action: proc (){.nimcall.}) =
+  template execute(cmd) =
+    curCaasCmd = cmd
+    processCmdLine(passCmd2, cmd)
+    action()
+
   let typ = getConfigVar("server.type")
   case typ
   of "stdin":
     while true:
       var line = stdin.readLine.string
       if line == "quit": quit()
-      processCmdLine(passCmd2, line)
+      execute line
-      action()
   of "tcp", "":
     var server = Socket()
     let p = getConfigVar("server.port")
@@ -75,8 +84,7 @@ proc serve*(action: proc (){.nimcall.}) =
     while true:
       accept(server, stdoutSocket)
       discard stdoutSocket.recvLine(inp)
-      processCmdLine(passCmd2, inp.string)
+      execute inp.string
-      action()
       stdoutSocket.send("\c\L")
       stdoutSocket.close()
   else:

compiler/syntaxes.nim

@@ -30,8 +30,8 @@ type
     parser*: TParser
 
 
-proc ParseFile*(filename: string): PNode{.procvar.}
+proc parseFile*(fileIdx: int32): PNode{.procvar.}
-proc openParsers*(p: var TParsers, filename: string, inputstream: PLLStream)
+proc openParsers*(p: var TParsers, fileIdx: int32, inputstream: PLLStream)
 proc closeParsers*(p: var TParsers)
 proc parseAll*(p: var TParsers): PNode
 proc parseTopLevelStmt*(p: var TParsers): PNode
@@ -40,14 +40,15 @@ proc parseTopLevelStmt*(p: var TParsers): PNode
 
 # implementation
 
-proc ParseFile(filename: string): PNode = 
+proc ParseFile(fileIdx: int32): PNode =
   var 
     p: TParsers
     f: tfile
+  let filename = fileIdx.toFilename
   if not open(f, filename):
     rawMessage(errCannotOpenFile, filename)
     return 
-  OpenParsers(p, filename, LLStreamOpen(f))
+  OpenParsers(p, fileIdx, LLStreamOpen(f))
   result = ParseAll(p)
   CloseParsers(p)
 
@@ -160,15 +161,16 @@ proc evalPipe(p: var TParsers, n: PNode, filename: string,
   else: 
     result = applyFilter(p, n, filename, result)
   
-proc openParsers(p: var TParsers, filename: string, inputstream: PLLStream) = 
+proc openParsers(p: var TParsers, fileIdx: int32, inputstream: PLLStream) = 
   var s: PLLStream
   p.skin = skinStandard
+  let filename = fileIdx.toFilename
   var pipe = parsePipe(filename, inputStream)
   if pipe != nil: s = evalPipe(p, pipe, filename, inputStream)
   else: s = inputStream
   case p.skin
   of skinStandard, skinBraces, skinEndX:
-    parser.openParser(p.parser, filename, s)
+    parser.openParser(p.parser, fileIdx, s)
   
 proc closeParsers(p: var TParsers) = 
   parser.closeParser(p.parser)