compiler bootstraps with new VM

compiler/main.nim

@@ -14,7 +14,7 @@ import
   os, condsyms, rodread, rodwrite, times,
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, jsgen, json, nversion,
-  platform, nimconf, importer, passaux, depends, evals, types, idgen,
+  platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
   tables, docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists,
   pretty
 

compiler/sem.nim

@@ -14,7 +14,7 @@ import
   wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
   magicsys, parser, nversion, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
-  semthreads, intsets, transf, evals, idgen, aliases, cgmeth, lambdalifting,
+  semthreads, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2
 
 # implementation
@@ -149,7 +149,8 @@ proc symNodeFromType(c: PContext, t: PType, info: TLineInfo): PNode =
   result = newSymNode(symFromType(t, info), info)
   result.typ = makeTypeDesc(c, t)
 
-proc createEvalContext(c: PContext, mode: TEvalMode): PEvalContext =
+when false:
+  proc createEvalContext(c: PContext, mode: TEvalMode): PEvalContext =
     result = newEvalContext(c.module, mode)
     result.getType = proc (n: PNode): PNode =
       var e = tryExpr(c, n)
@@ -163,10 +164,10 @@ proc createEvalContext(c: PContext, mode: TEvalMode): PEvalContext =
     result.handleIsOperator = proc (n: PNode): PNode =
       result = IsOpImpl(c, n)
 
-proc evalConstExpr(c: PContext, module: PSym, e: PNode): PNode = 
+  proc evalConstExpr(c: PContext, module: PSym, e: PNode): PNode = 
     result = evalConstExprAux(c.createEvalContext(emConst), module, nil, e)
 
-proc evalStaticExpr(c: PContext, module: PSym, e: PNode, prc: PSym): PNode = 
+  proc evalStaticExpr(c: PContext, module: PSym, e: PNode, prc: PSym): PNode = 
     result = evalConstExprAux(c.createEvalContext(emStatic), module, prc, e)
 
 proc semConstExpr(c: PContext, n: PNode): PNode =
@@ -176,7 +177,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
     return n
   result = getConstExpr(c.module, e)
   if result == nil:
-    result = evalConstExpr(c, c.module, e)
+    result = evalConstExpr(c.module, e)
     if result == nil or result.kind == nkEmpty:
       if e.info != n.info:
         pushInfoContext(n.info)
@@ -222,10 +223,10 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
   if sym == c.p.owner:
     GlobalError(n.info, errRecursiveDependencyX, sym.name.s)
 
-  if c.evalContext == nil:
+  #if c.evalContext == nil:
-    c.evalContext = c.createEvalContext(emStatic)
+  #  c.evalContext = c.createEvalContext(emStatic)
 
-  result = evalMacroCall(c.evalContext, n, nOrig, sym)
+  result = evalMacroCall(c.module, n, nOrig, sym)
   if semCheck: result = semAfterMacroCall(c, result, sym)
 
 proc forceBool(c: PContext, n: PNode): PNode = 

compiler/semdata.nim

@@ -13,7 +13,7 @@ import
   strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab,
   wordrecg, 
   ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, 
-  magicsys, nversion, nimsets, parser, times, passes, rodread, evals
+  magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef
 
 type 
   TOptionEntry* = object of lists.TListEntry # entries to put on a

compiler/semexprs.nim

@@ -631,18 +631,18 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       call.add(a)
     #echo "NOW evaluating at compile time: ", call.renderTree
     if sfCompileTime in callee.flags:
-      result = evalStaticExpr(c, c.module, call, c.p.owner)
+      result = evalStaticExpr(c.module, call, c.p.owner)
       if result.isNil: 
         LocalError(n.info, errCannotInterpretNodeX, renderTree(call))
     else:
-      result = evalConstExpr(c, c.module, call)
+      result = evalConstExpr(c.module, call)
       if result.isNil: result = n
     #if result != n:
     #  echo "SUCCESS evaluated at compile time: ", call.renderTree
 
 proc semStaticExpr(c: PContext, n: PNode): PNode =
   let a = semExpr(c, n.sons[0])
-  result = evalStaticExpr(c, c.module, a, c.p.owner)
+  result = evalStaticExpr(c.module, a, c.p.owner)
   if result.isNil:
     LocalError(n.info, errCannotInterpretNodeX, renderTree(n))
     result = emptyNode

compiler/semmagic.nim

@@ -32,6 +32,23 @@ proc semInstantiationInfo(c: PContext, n: PNode): PNode =
   result.add(filename)
   result.add(line)
 
+ 
+proc evalTypeTrait(trait, operand: PNode, context: PSym): PNode =
+  InternalAssert operand.kind == nkSym
+
+  let typ = operand.sym.typ.skipTypes({tyTypeDesc})
+  case trait.sym.name.s.normalize
+  of "name":
+    result = newStrNode(nkStrLit, typ.typeToString(preferName))
+    result.typ = newType(tyString, context)
+    result.info = trait.info
+  of "arity":    
+    result = newIntNode(nkIntLit, typ.n.len-1)
+    result.typ = newType(tyInt, context)
+    result.info = trait.info
+  else:
+    internalAssert false
+
 proc semTypeTraits(c: PContext, n: PNode): PNode =
   checkMinSonsLen(n, 2)
   internalAssert n.sons[1].kind == nkSym

compiler/semstmts.nim

@@ -1152,14 +1152,14 @@ proc setLine(n: PNode, info: TLineInfo) =
 proc semPragmaBlock(c: PContext, n: PNode): PNode =
   let pragmaList = n.sons[0]
   pragma(c, nil, pragmaList, exprPragmas)
-  result = semStmt(c, n.sons[1])
+  result = semExpr(c, n.sons[1])
   for i in 0 .. <pragmaList.len:
     if whichPragma(pragmaList.sons[i]) == wLine:
       setLine(result, pragmaList.sons[i].info)
 
 proc semStaticStmt(c: PContext, n: PNode): PNode =
   let a = semStmt(c, n.sons[0])
-  result = evalStaticExpr(c, c.module, a, c.p.owner)
+  result = evalStaticExpr(c.module, a, c.p.owner)
   if result.isNil:
     LocalError(n.info, errCannotInterpretNodeX, renderTree(n))
     result = emptyNode

compiler/vm.nim

@@ -12,7 +12,7 @@
 
 import
   strutils, ast, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned,
-  parser, vmdeps, idents
+  parser, vmdeps, idents, trees, renderer
 
 from semfold import leValueConv, ordinalValToString
 
@@ -231,13 +231,19 @@ proc compile(c: PCtx, s: PSym): int =
   result = vmgen.genProc(c, s)
   #c.echoCode
 
-proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
+proc regsContents(regs: TNodeSeq) =
+  for i in 0.. <regs.len:
+    echo "Register ", i
+    #debug regs[i]
+
+proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
   var pc = start
   var tos = tos
   var regs: TNodeSeq # alias to tos.slots for performance
   move(regs, tos.slots)
+  #echo "NEW RUN ------------------------"
   while true:
-    {.computedGoto.}
+    #{.computedGoto.}
     let instr = c.code[pc]
     let ra = instr.regA
     #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra
@@ -248,12 +254,15 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
       pc = tos.comesFrom
       tos = tos.next
       let retVal = regs[0]
-      if tos.isNil: return retVal
+      if tos.isNil: 
+        #echo "RET ", retVal.rendertree
+        return retVal
       
       move(regs, tos.slots)
       assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
       if c.code[pc].opcode == opcIndCallAsgn:
         regs[c.code[pc].regA] = retVal
+        #echo "RET2 ", retVal.rendertree, " ", c.code[pc].regA
     of opcYldYoid: assert false
     of opcYldVal: assert false
     of opcAsgnInt:
@@ -508,7 +517,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
     of opcEcho:
       let rb = instr.regB
       for i in ra..ra+rb-1:
-        if regs[i].kind != nkStrLit: debug regs[i]
+        #if regs[i].kind != nkStrLit: debug regs[i]
         write(stdout, regs[i].strVal)
       writeln(stdout, "")
     of opcContainsSet:
@@ -535,6 +544,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
       let rc = instr.regC
       let prc = regs[rb].sym
       let newPc = compile(c, prc)
+      #echo "new pc ", newPc, " calling: ", prc.name.s
       var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos)
       newSeq(newFrame.slots, prc.position)
       if not isEmptyType(prc.typ.sons[0]):
@@ -841,10 +851,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame) =
       regs[ra].strVal = typ.typeToString(preferExported)
     inc pc
 
-proc execute(c: PCtx, start: int) =
+proc execute(c: PCtx, start: int): PNode =
   var tos = PStackFrame(prc: nil, comesFrom: 0, next: nil)
   newSeq(tos.slots, c.prc.maxSlots)
-  rawExecute(c, start, tos)
+  for i in 0 .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  result = rawExecute(c, start, tos)
 
 proc evalStmt*(c: PCtx, n: PNode) =
   let start = genStmt(c, n)
@@ -858,11 +869,24 @@ proc evalExpr*(c: PCtx, n: PNode): PNode =
   assert c.code[start].opcode != opcEof
   result = execute(c, start)
 
+# for now we share the 'globals' environment. XXX Coming soon: An API for
+# storing&loading the 'globals' environment to get what a component system
+# requires.
+var
+  globalCtx: PCtx
+
+proc setupGlobalCtx(module: PSym) =
+  if globalCtx.isNil: globalCtx = newCtx(module)
+  else: refresh(globalCtx, module)
+
 proc myOpen(module: PSym): PPassContext =
   #var c = newEvalContext(module, emRepl)
   #c.features = {allowCast, allowFFI, allowInfiniteLoops}
   #pushStackFrame(c, newStackFrame())
-  result = newCtx(module)
+
+  # XXX produce a new 'globals' environment here:
+  setupGlobalCtx(module)
+  result = globalCtx
 
 var oldErrorCount: int
 
@@ -875,17 +899,17 @@ proc myProcess(c: PPassContext, n: PNode): PNode =
     result = n
   oldErrorCount = msgs.gErrorCounter
 
-const vmPass* = makePass(myOpen, nil, myProcess, myProcess)
+const evalPass* = makePass(myOpen, nil, myProcess, myProcess)
 
-proc evalConstExprAux(module, prc: PSym, e: PNode, mode: TEvalMode): PNode = 
+proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode =
-  var p = newCtx(module)
+  setupGlobalCtx(module)
-  var s = newStackFrame()
+  var c = globalCtx
-  s.call = e
+  let start = genExpr(c, n)
-  s.prc = prc
+  assert c.code[start].opcode != opcEof
-  pushStackFrame(p, s)
+  var tos = PStackFrame(prc: prc, comesFrom: 0, next: nil)
-  result = tryEval(p, e)
+  newSeq(tos.slots, c.prc.maxSlots)
-  if result != nil and result.kind == nkExceptBranch: result = nil
+  for i in 0 .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
-  popStackFrame(p)
+  result = rawExecute(c, start, tos)
 
 proc evalConstExpr*(module: PSym, e: PNode): PNode = 
   result = evalConstExprAux(module, nil, e, emConst)
@@ -897,15 +921,18 @@ proc setupMacroParam(x: PNode): PNode =
   result = x
   if result.kind in {nkHiddenSubConv, nkHiddenStdConv}: result = result.sons[1]
 
-proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
+var evalMacroCounter: int
+
+proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
   # XXX GlobalError() is ugly here, but I don't know a better solution for now
-  inc(evalTemplateCounter)
+  inc(evalMacroCounter)
-  if evalTemplateCounter > 100:
+  if evalMacroCounter > 100:
     GlobalError(n.info, errTemplateInstantiationTooNested)
+  setupGlobalCtx(module)
+  var c = globalCtx
 
   c.callsite = nOrig
-  let body = optBody(c, sym)
+  let start = genProc(c, sym)
-  let start = genStmt(c, body)
 
   var tos = PStackFrame(prc: sym, comesFrom: 0, next: nil)
   newSeq(tos.slots, c.prc.maxSlots)
@@ -917,8 +944,9 @@ proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode =
   tos.slots[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
   # setup parameters:
   for i in 1 .. < L: tos.slots[i] = setupMacroParam(n.sons[i])
-  rawExecute(c, start, tos)
+  # temporary storage:
-  result = tos.slots[0]
+  for i in L .. <c.prc.maxSlots: tos.slots[i] = newNode(nkEmpty)
+  result = rawExecute(c, start, tos)
   if cyclicTree(result): GlobalError(n.info, errCyclicTree)
-  dec(evalTemplateCounter)
+  dec(evalMacroCounter)
   c.callsite = nil

compiler/vmdef.nim

@@ -165,12 +165,30 @@ type
 
   PEvalContext* = PCtx
 
+  TEvalMode* = enum           ## reason for evaluation
+    emRepl,                   ## evaluate because in REPL mode
+    emConst,                  ## evaluate for 'const' according to spec
+    emOptimize,               ## evaluate for optimization purposes (same as
+                              ## emConst?)
+    emStatic                  ## evaluate for enforced compile time eval
+                              ## ('static' context)
+
+  TSandboxFlag* = enum        ## what the evaluation engine should allow
+    allowCast,                ## allow unsafe language feature: 'cast'
+    allowFFI,                 ## allow the FFI
+    allowInfiniteLoops        ## allow endless loops
+  TSandboxFlags* = set[TSandboxFlag]
+
   
 proc newCtx*(module: PSym): PCtx =
   PCtx(code: @[], debug: @[],
     globals: newNode(nkStmtList), constants: newNode(nkStmtList), types: @[],
     prc: PProc(blocks: @[]), module: module)
 
+proc refresh*(c: PCtx, module: PSym) =
+  c.module = module
+  c.prc = PProc(blocks: @[])
+
 const
   firstABxInstr* = opcTJmp
   largeInstrs* = { # instructions which use 2 int32s instead of 1:

compiler/vmgen.nim

@@ -45,7 +45,7 @@ proc codeListing(c: PCtx, result: var string) =
     result.add("\n")
     inc i
 
-proc echoCode*(c: PCtx) =
+proc echoCode*(c: PCtx) {.deprecated.} =
   var buf = ""
   codeListing(c, buf)
   echo buf
@@ -494,7 +494,9 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
     genBinaryABC(c, n, dest, opc)
 
 proc unused(n: PNode; x: TDest) {.inline.} =
-  if x >= 0: InternalError(n.info, "not unused")
+  if x >= 0: 
+    #debug(n)
+    InternalError(n.info, "not unused")
 
 proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =  
   let tmp = c.genx(arg)
@@ -508,7 +510,7 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
   let tmp = c.genx(n.sons[1])
   if dest < 0: dest = c.getTemp(n.typ)
   c.genSetType(n.sons[1], tmp)
-  c.gABC(n, opc, dest, tmp)
+  c.gABC(n, opcCard, dest, tmp)
   c.freeTemp(tmp)
 
 proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
@@ -701,7 +703,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
   of mTypeTrait: 
     let tmp = c.genx(n.sons[1])
     if dest < 0: dest = c.getTemp(n.typ)
-    c.gABx(n, opcSetType, tmp, c.genType(n.sons[1]))
+    c.gABx(n, opcSetType, tmp, c.genType(n.sons[1].typ))
     c.gABC(n, opcTypeTrait, dest, tmp)
     c.freeTemp(tmp)
   of mIs:
@@ -1259,7 +1261,14 @@ proc optimizeJumps(c: PCtx; start: int) =
 proc genProc(c: PCtx; s: PSym): int =
   let x = s.ast.sons[optimizedCodePos]
   if x.kind == nkEmpty:
-    c.removeLastEof
+    #echo "GENERATING CODE FOR ", s.name.s
+    let last = c.code.len-1
+    var eofInstr: TInstr
+    if last >= 0 and c.code[last].opcode == opcEof:
+      eofInstr = c.code[last]
+      c.code.setLen(last)
+      c.debug.setLen(last)
+    #c.removeLastEof
     result = c.code.len+1 # skip the jump instruction
     s.ast.sons[optimizedCodePos] = newIntNode(nkIntLit, result)
     # thanks to the jmp we can add top level statements easily and also nest
@@ -1275,9 +1284,9 @@ proc genProc(c: PCtx; s: PSym): int =
     # generate final 'return' statement:
     c.gABC(body, opcRet)
     c.patch(procStart)
-    c.gABC(body, opcEof)
+    
+    c.gABC(body, opcEof, eofInstr.regA)
     s.position = c.prc.maxSlots
     c.prc = oldPrc
-    #c.echoCode
   else:
     result = x.intVal.int

todo.txt

@@ -2,8 +2,9 @@ version 0.9.4
 =============
 
 - new VM:
-  - implement the glue to replace evals.nim
+  - new VM requires lambda lifting
-  - implement missing magics
+  - codegen for computed goto still wrong
+  - test and activate the jump optimizer
   - implement overflow checking
   - implement the FFI
 

web/news.txt

@@ -41,6 +41,9 @@ Compiler Additions
   over the generated code.
 - The compiler now supports a ``computedGoto`` pragma to support very fast
   dispatching for interpreters and the like.
+- The old evaluation engine has been replaced by a proper register based
+  virtual machine. This fixes numerous bugs for ``nimrod i`` and for macro
+  evaluation.
 
 
 Language Additions
@@ -55,7 +58,7 @@ Language Additions
   OOP-like syntactic sugar.
 - Added ``delegator pragma`` for handling calls to missing procs and fields at
   compile-time.
-- Support for user-defined type classes have been added.
+- Support for user-defined type classes has been added.
 
 
 Tools improvements