caas is now drivable through stdin

* added idetools --eval
* streams.readLine recognises and applies the backspace character

compiler/cgen.nim

@@ -19,7 +19,6 @@ import
 when options.hasTinyCBackend:
   import tccgen
 
-proc cgenPass*(): TPass
 # implementation
 
 var
@@ -1157,10 +1156,5 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     writeMapping(gMapping)
     if generatedHeader != nil: writeHeader(generatedHeader)
 
-proc cgenPass(): TPass = 
+const cgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-  initPass(result)
-  result.open = myOpen
-  result.openCached = myOpenCached
-  result.process = myProcess
-  result.close = myClose
 

compiler/commands.nim

@@ -458,6 +458,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "def":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optDef)
+  of "eval":
+    expectArg(switch, arg, pass, info)
+    gEvalExpr = arg
   of "context":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optContext)

compiler/depends.nim

@@ -12,7 +12,6 @@
 import 
   os, options, ast, astalgo, msgs, ropes, idents, passes, importer
 
-proc genDependPass*(): TPass
 proc generateDot*(project: string)
 
 type 
@@ -55,7 +54,5 @@ proc myOpen(module: PSym, filename: string): PPassContext =
   g.filename = filename
   result = g
 
-proc gendependPass(): TPass = 
+const gendependPass* = makePass(open = myOpen, process = addDotDependency)
-  initPass(result)
+
-  result.open = myOpen
-  result.process = addDotDependency

compiler/docgen2.nim

@@ -45,11 +45,7 @@ proc myOpen(module: PSym, filename: string): PPassContext =
   g.doc = d
   result = g
 
-proc docgen2Pass*(): TPass = 
+const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close)
-  initPass(result)
-  result.open = myOpen
-  result.process = processNode
-  result.close = close
 
 proc finishDoc2Pass*(project: string) = 
   nil

compiler/ecmasgen.nim

@@ -17,7 +17,6 @@ import
   times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
   intsets, cgmeth
 
-proc ecmasgenPass*(): TPass
 # implementation
 
 type 
@@ -1620,9 +1619,4 @@ proc myOpenCached(s: PSym, filename: string, rd: PRodReader): PPassContext =
 proc myOpen(s: PSym, filename: string): PPassContext = 
   result = newModule(s, filename)
 
-proc ecmasgenPass(): TPass = 
+const ecmasgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-  InitPass(result)
-  result.open = myOpen
-  result.close = myClose
-  result.openCached = myOpenCached
-  result.process = myProcess

compiler/evals.nim

@@ -1440,9 +1440,5 @@ proc myOpen(module: PSym, filename: string): PPassContext =
 proc myProcess(c: PPassContext, n: PNode): PNode = 
   result = eval(PEvalContext(c), n)
 
-proc evalPass*(): TPass = 
+const evalPass* = makePass(myOpen, nil, myProcess, myProcess)
-  initPass(result)
-  result.open = myOpen
-  result.close = myProcess
-  result.process = myProcess
 

compiler/main.nim

@@ -16,7 +16,7 @@ import
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, ecmasgen,
   platform, nimconf, importer, passaux, depends, evals, types, idgen,
-  tables, docgen2, service
+  tables, docgen2, service, magicsys, parser
 
 const
   has_LLVM_Backend = false
@@ -89,22 +89,30 @@ proc `==^`(a, b: string): bool =
   except EOS:
     result = false
 
+proc compileSystemModule =
+  if magicsys.SystemModule == nil:
+    discard CompileModule(options.libpath /"system", {sfSystemModule})
+
 proc CompileProject(projectFile = gProjectFull) =
   let systemFile = options.libpath / "system"
   if projectFile.addFileExt(nimExt) ==^ systemFile.addFileExt(nimExt):
     discard CompileModule(projectFile, {sfMainModule, sfSystemModule})
   else:
-    discard CompileModule(systemFile, {sfSystemModule})
+    compileSystemModule()
     discard CompileModule(projectFile, {sfMainModule})
 
+proc rodPass =
+  if optSymbolFiles in gGlobalOptions:
+    registerPass(rodwritePass)
+
 proc semanticPasses =
-  registerPass(verbosePass())
+  registerPass verbosePass
-  registerPass(sem.semPass())
+  registerPass semPass
 
 proc CommandGenDepend =
   semanticPasses()
-  registerPass(genDependPass())
+  registerPass(genDependPass)
-  registerPass(cleanupPass())
+  registerPass(cleanupPass)
   compileProject()
   generateDot(gProjectFull)
   execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") &
@@ -113,21 +121,21 @@ proc CommandGenDepend =
 proc CommandCheck =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()            # use an empty backend for semantic checking only
-  registerPass(rodwrite.rodwritePass())
+  rodPass()
   compileProject(mainCommandArg())
 
 proc CommandDoc2 =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
-  registerPass(docgen2Pass())
+  registerPass(docgen2Pass)
   #registerPass(cleanupPass())
   compileProject(mainCommandArg())
   finishDoc2Pass(gProjectFull)
 
 proc CommandCompileToC =
   semanticPasses()
-  registerPass(cgen.cgenPass())
+  registerPass(cgenPass)
-  registerPass(rodwrite.rodwritePass())
+  rodPass()
   #registerPass(cleanupPass())
   compileProject()
   if gCmd != cmdRun:
@@ -137,7 +145,7 @@ when has_LLVM_Backend:
   proc CommandCompileToLLVM =
     semanticPasses()
     registerPass(llvmgen.llvmgenPass())
-    registerPass(rodwrite.rodwritePass())
+    rodPass()
     #registerPass(cleanupPass())
     compileProject()
 
@@ -148,27 +156,50 @@ proc CommandCompileToEcmaScript =
   DefineSymbol("nimrod") # 'nimrod' is always defined
   DefineSymbol("ecmascript")
   semanticPasses()
-  registerPass(ecmasgenPass())
+  registerPass(ecmasgenPass)
   compileProject()
 
-proc CommandInteractive =
+proc InteractivePasses =
-  msgs.gErrorMax = high(int)  # do not stop after first error
   incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
   DefineSymbol("nimrodvm")
-  registerPass(verbosePass())
+  registerPass(verbosePass)
-  registerPass(sem.semPass())
+  registerPass(semPass)
-  registerPass(evals.evalPass()) # load system module:
+  registerPass(evalPass)
-  discard CompileModule(options.libpath /"system", {sfSystemModule})
+
+var stdinModule: PSym
+proc makeStdinModule: PSym =
+  if stdinModule == nil:
+    stdinModule = newModule("stdin")
+    stdinModule.id = getID()
+  result = stdinModule
+
+proc CommandInteractive =
+  msgs.gErrorMax = high(int)  # do not stop after first error
+  InteractivePasses()
+  compileSystemModule()
   if commandArgs.len > 0:
     discard CompileModule(mainCommandArg(), {})
   else:
-    var m = newModule("stdin")
+    var m = makeStdinModule()
-    m.id = getID()
     incl(m.flags, sfMainModule)
     processModule(m, "stdin", LLStreamOpenStdIn(), nil)
 
+const evalPasses = [verbosePass, semPass, evalPass]
+
+proc evalNim(nodes: PNode, module: PSym, filename: string) =
+  # we don't want to mess with gPasses here, because in nimrod serve
+  # scenario, it may be set up properly for normal cgenPass() compilation
+  carryPasses(nodes, module, filename, evalPasses)
+
+proc commandEval(exp: string) =
+  if SystemModule == nil:
+    InteractivePasses()
+    compileSystemModule()
+  var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
+  evalNim(echoExp.parseString, makeStdinModule(), "stdin")
+
 proc CommandPretty =
   var module = parseFile(addFileExt(mainCommandArg(), NimExt))
   if module != nil: 
@@ -194,7 +225,7 @@ proc CommandScan =
 proc CommandSuggest =
   msgs.gErrorMax = high(int)  # do not stop after first error
   semanticPasses()
-  registerPass(rodwrite.rodwritePass())
+  rodPass()
   compileProject()
 
 proc wantMainModule =
@@ -202,6 +233,8 @@ proc wantMainModule =
     Fatal(gCmdLineInfo, errCommandExpectsFilename)
   
 proc MainCommand =
+  # In "nimrod serve" scenario, each command must reset the registered passes
+  clearPasses()
   appendStr(searchPaths, options.libpath)
   if gProjectFull.len != 0:
     # current path is always looked first for modules
@@ -299,20 +332,20 @@ proc MainCommand =
   of "i": 
     gCmd = cmdInteractive
     CommandInteractive()
+  of "e":
+    # XXX: temporary command for easier testing
+    commandEval(mainCommandArg())
   of "idetools":
     gCmd = cmdIdeTools
-    wantMainModule()
+    if gEvalExpr != "":
-    CommandSuggest()
+      commandEval(gEvalExpr)
+    else:
+      wantMainModule()
+      CommandSuggest()
   of "serve":
     gCmd = cmdIdeTools
-    msgs.gErrorMax = high(int)  # do not stop after first error
+    msgs.gErrorMax = high(int)  # do not stop after first error     
-    semanticPasses()
+    serve(MainCommand)
-    # no need to write rod files and would slow down things:
+    
-    #registerPass(rodwrite.rodwritePass())
-    discard CompileModule(options.libpath / "system", {sfSystemModule})
-    service.serve(proc () =
-      let projectFile = mainCommandArg()
-      discard CompileModule(projectFile, {sfMainModule})
-    )
   else: rawMessage(errInvalidCommandX, command)
 

compiler/nimconf.nim

@@ -235,15 +235,16 @@ proc LoadConfigs*(cfg: string) =
   if optSkipParentConfigFiles notin gGlobalOptions:
     for dir in parentDirs(pd, fromRoot=true, inclusive=false):
       readConfigFile(dir / cfg)
-    
+  
-  if optSkipProjConfigFile notin gGlobalOptions and gProjectName.len != 0:
+  if optSkipProjConfigFile notin gGlobalOptions:
     readConfigFile(pd / cfg)
     
-    var conffile = changeFileExt(gProjectFull, "cfg")
+    if gProjectName.len != 0:
-    if conffile != pd / cfg and existsFile(conffile):
+      var conffile = changeFileExt(gProjectFull, "cfg")
-      readConfigFile(conffile)
+      if conffile != pd / cfg and existsFile(conffile):
-      rawMessage(warnConfigDeprecated, conffile)
+        readConfigFile(conffile)
-    
+        rawMessage(warnConfigDeprecated, conffile)
-    # new project wide config file:
+      
-    readConfigFile(changeFileExt(gProjectFull, "nimrod.cfg"))
+      # new project wide config file:
+      readConfigFile(changeFileExt(gProjectFull, "nimrod.cfg"))
  

compiler/nimrod.cfg

@@ -1,9 +1,12 @@
 # Special configuration file for the Nimrod project
 
---hint[XDeclaredButNotUsed]=off
+mainModule:"nimrod.nim"
-path="llvm"
-path="$projectPath/.."
 
-path="$nimrod/packages/docutils"
+hint[XDeclaredButNotUsed]:off
+path:"llvm"
+path:"$projectPath/.."
+
+path:"$nimrod/packages/docutils"
+
+define:booting
 
---define:booting

compiler/options.nim

@@ -94,8 +94,8 @@ var
   gCmd*: TCommands = cmdNone  # the command
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
-
   gWholeProject*: bool # for 'doc2': output any dependency
+  gEvalExpr*: string          # expression for idetools --eval
   
 const 
   genSubDir* = "nimcache"

compiler/passaux.nim

@@ -26,10 +26,7 @@ proc verboseProcess(context: PPassContext, n: PNode): PNode =
     incl(msgs.gNotes, hintProcessing)
     Message(n.info, hintProcessing, $idgen.gBackendId)
   
-proc verbosePass*(): TPass = 
+const verbosePass* = makePass(open = verboseOpen, process = verboseProcess)
-  initPass(result)
-  result.open = verboseOpen
-  result.process = verboseProcess
 
 proc cleanUp(c: PPassContext, n: PNode): PNode = 
   result = n                  
@@ -46,7 +43,5 @@ proc cleanUp(c: PPassContext, n: PNode): PNode =
   else: 
     nil
 
-proc cleanupPass*(): TPass = 
+const cleanupPass* = makePass(process = cleanUp, close = cleanUp)
-  initPass(result)
+
-  result.process = cleanUp
-  result.close = cleanUp

compiler/passes.nim

@@ -18,21 +18,34 @@ import
 type  
   TPassContext* = object of TObject # the pass's context
     fromCache*: bool  # true if created by "openCached"
-    
+   
   PPassContext* = ref TPassContext
-  TPass* = tuple[
+
-    open: proc (module: PSym, filename: string): PPassContext {.nimcall.},
+  TPassOpen* = proc (module: PSym, filename: string): PPassContext {.nimcall.}
-    openCached: proc (module: PSym, filename: string,
+  TPassOpenCached* = proc (module: PSym, filename: string,
-                     rd: PRodReader): PPassContext {.nimcall.},
+                           rd: PRodReader): PPassContext {.nimcall.}
-    close: proc (p: PPassContext, n: PNode): PNode {.nimcall.},
+  TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.}
-    process: proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}]
+  TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.}
-    
+
+  TPass* = tuple[open: TPassOpen, openCached: TPassOpenCached,
+                 process: TPassProcess, close: TPassClose]
+
+  TPassData* = tuple[input: PNode, closeOutput: Pnode]
+  TPasses* = openarray[TPass]
+
 # a pass is a tuple of procedure vars ``TPass.close`` may produce additional 
 # nodes. These are passed to the other close procedures. 
 # This mechanism used to be used for the instantiation of generics.
 
-proc registerPass*(p: TPass)
+proc makePass*(open: TPassOpen = nil,
-proc initPass*(p: var TPass)
+               openCached: TPassOpenCached = nil,
+               process: TPassProcess = nil,
+               close: TPassClose = nil): TPass =
+  result.open = open
+  result.openCached = openCached
+  result.close = close
+  result.process = process
+
   # This implements a memory preserving scheme: Top level statements are
   # processed in a pipeline. The compiler never looks at a whole module
   # any longer. However, this is simple to change, as new passes may perform
@@ -74,12 +87,28 @@ type
 
 var 
   gPasses: array[0..maxPasses - 1, TPass]
-  gPassesLen: int
+  gPassesLen*: int
 
-proc registerPass(p: TPass) = 
+proc clearPasses* =
+  gPassesLen = 0
+
+proc registerPass*(p: TPass) = 
   gPasses[gPassesLen] = p
   inc(gPassesLen)
 
+proc carryPass*(p: TPass, module: PSym, filename: string,
+                m: TPassData): TPassData =
+  var c = p.open(module, filename)
+  result.input = p.process(c, m.input)
+  result.closeOutput = if p.close != nil: p.close(c, m.closeOutput)
+                       else: m.closeOutput
+
+proc carryPasses*(nodes: PNode, module: PSym, file: string, passes: TPasses) =
+  var passdata: TPassData
+  passdata.input = nodes
+  for pass in passes:
+    passdata = carryPass(pass, module, file, passdata)
+
 proc openPasses(a: var TPassContextArray, module: PSym, filename: string) = 
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].open): 
@@ -175,8 +204,3 @@ proc processModule(module: PSym, filename: string, stream: PLLStream,
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)
 
-proc initPass(p: var TPass) = 
-  p.open = nil
-  p.openCached = nil
-  p.close = nil
-  p.process = nil

compiler/rodwrite.nim

@@ -15,7 +15,6 @@ import
   intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
   condsyms, ropes, idents, crc, rodread, passes, importer, idgen, rodutils
 
-proc rodwritePass*(): TPass
 # implementation
 
 type 
@@ -583,10 +582,5 @@ proc myClose(c: PPassContext, n: PNode): PNode =
   writeRod(w)
   idgen.saveMaxIds(options.gProjectPath / options.gProjectName)
 
-proc rodwritePass(): TPass = 
+const rodwritePass* = makePass(open = myOpen, close = myClose, process = process)
-  initPass(result)
-  if optSymbolFiles in gGlobalOptions: 
-    result.open = myOpen
-    result.close = myClose
-    result.process = process
 

compiler/sem.nim

@@ -17,7 +17,6 @@ import
   semthreads, intsets, transf, evals, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2
 
-proc semPass*(): TPass
 # implementation
 
 type 
@@ -288,9 +287,5 @@ proc myClose(context: PPassContext, n: PNode): PNode =
   popOwner()
   popProcCon(c)
 
-proc semPass(): TPass =
+const semPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-  initPass(result)
+
-  result.open = myOpen
-  result.openCached = myOpenCached
-  result.close = myClose
-  result.process = myProcess

compiler/service.nim

@@ -56,17 +56,30 @@ proc ProcessCmdLine*(pass: TCmdLinePass, cmd: string) =
       rawMessage(errArgsNeedRunOption, [])
 
 proc serve*(action: proc (){.nimcall.}) =
-  var server = Socket()
+  let typ = getConfigVar("server.type")
-  let p = getConfigVar("server.port")
+  case typ
-  let port = if p.len > 0: parseInt(p).TPort else: 6000.TPort
+  of "stdin":
-  server.bindAddr(port, getConfigVar("server.address"))
+    while true:
-  var inp = "".TaintedString
+      var line = stdin.readLine.string
-  server.listen()
+      if line == "quit": quit()
-  new(stdoutSocket)
+      processCmdLine(passCmd2, line)
-  while true:
+      action()
-    accept(server, stdoutSocket)
+  of "tcp", "":
-    discard stdoutSocket.recvLine(inp)
+    var server = Socket()
-    processCmdLine(passCmd2, inp.string)
+    let p = getConfigVar("server.port")
-    action()
+    let port = if p.len > 0: parseInt(p).TPort else: 6000.TPort
-    stdoutSocket.send("\c\L")
+    server.bindAddr(port, getConfigVar("server.address"))
-    stdoutSocket.close()
+    var inp = "".TaintedString
+    server.listen()
+    new(stdoutSocket)
+    while true:
+      accept(server, stdoutSocket)
+      discard stdoutSocket.recvLine(inp)
+      processCmdLine(passCmd2, inp.string)
+      action()
+      stdoutSocket.send("\c\L")
+      stdoutSocket.close()
+  else:
+    echo "Invalid server.type:", typ
+    quit 1
+

compiler/transf.nim

@@ -696,12 +696,8 @@ when false:
     result = openTransf(module, filename)
     for m in items(rd.methods): methodDef(m, true)
 
-  proc transfPass(): TPass = 
+  const transfPass* = makePass(openTransf, openTransfCached,
-    initPass(result)
+    processTransf, processTransf) # we need to process generics too!
-    result.open = openTransf
-    result.openCached = openTransfCached
-    result.process = processTransf
-    result.close = processTransf # we need to process generics too!
   
 proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
   if nfTransf in n.flags or prc.kind in {skTemplate, skMacro}:

doc/advopt.txt

@@ -16,6 +16,7 @@ Advanced commands:
     --def                   list all possible definitions at position
     --context               list possible invokation context
     --usages                list all usages of the symbol at position
+    --eval                  evaluates an expression
 
 Advanced options:
   -m, --mainmodule:FILE     set the project main module

lib/pure/streams.nim

@@ -180,8 +180,12 @@ proc readLine*(s: PStream): TaintedString =
     if c == '\c': 
       c = readChar(s)
       break
-    elif c == '\L' or c == '\0': break
+    if c == '\b':
-    result.string.add(c)
+      result.string.setLen(result.len - 1)
+    elif c == '\L' or c == '\0':
+      break
+    else:
+      result.string.add(c)
 
 type
   PStringStream* = ref TStringStream ## a stream that encapsulates a string