Rework exception handling in the VM (#10544)

* Rework exception handling in the VM

Make the safepoint handling more precise and less forgiving.
The new code is clearer and more commented.
Perform cleanup on `return`.
The no-exception-thrown case in a try block should be slightly faster
since we don't parse the whole set of exceptions every time.
More tests.

* Fix silly error that broke a few tests

* Testament doesn't like files having the same name

* Remove test case that failed compilation to js

compiler/vm.nim

@@ -23,7 +23,7 @@ from evaltempl import evalTemplate
 from modulegraphs import ModuleGraph, PPassContext
 
 const
-  traceCode = debugEchoCode
+  traceCode = defined(nimVMDebug)
 
 when hasFFI:
   import evalffi
@@ -259,64 +259,101 @@ proc pushSafePoint(f: PStackFrame; pc: int) =
   f.safePoints.add(pc)
 
 proc popSafePoint(f: PStackFrame) =
-  # XXX this needs a proper fix!
+  discard f.safePoints.pop()
-  if f.safePoints.len > 0:
-    discard f.safePoints.pop()
 
-proc cleanUpOnException(c: PCtx; tos: PStackFrame):
+type
-                                              tuple[pc: int, f: PStackFrame] =
+  ExceptionGoto = enum
-  let raisedType = c.currentExceptionA.typ.skipTypes(abstractPtrs)
+    ExceptionGotoHandler,
-  var f = tos
+    ExceptionGotoFinally,
-  while true:
+    ExceptionGotoUnhandled
-    while f.safePoints.len == 0:
-      f = f.next
-      if f.isNil: return (-1, nil)
-    var pc2 = f.safePoints[f.safePoints.high]
 
-    var nextExceptOrFinally = -1
+proc findExceptionHandler(c: PCtx, f: PStackFrame, exc: PNode):
-    if c.code[pc2].opcode == opcExcept:
+  tuple[why: ExceptionGoto, where: int] =
-      nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess
+  let raisedType = exc.typ.skipTypes(abstractPtrs)
-      inc pc2
-    while c.code[pc2].opcode == opcExcept:
-      let excIndex = c.code[pc2].regBx-wordExcess
-      let exceptType = if excIndex > 0: c.types[excIndex].skipTypes(
-                          abstractPtrs)
-                       else: nil
-      #echo typeToString(exceptType), " ", typeToString(raisedType)
-      if exceptType.isNil or inheritanceDiff(raisedType, exceptType) <= 0:
-        # mark exception as handled but keep it in B for
-        # the getCurrentException() builtin:
-        c.currentExceptionB = c.currentExceptionA
-        c.currentExceptionA = nil
-        # execute the corresponding handler:
-        while c.code[pc2].opcode == opcExcept: inc pc2
-        discard f.safePoints.pop
-        return (pc2, f)
-      inc pc2
-      if c.code[pc2].opcode != opcExcept and nextExceptOrFinally >= 0:
-        # we're at the end of the *except list*, but maybe there is another
-        # *except branch*?
-        pc2 = nextExceptOrFinally+1
-        if c.code[pc2].opcode == opcExcept:
-          nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess
 
-    if nextExceptOrFinally >= 0:
+  while f.safePoints.len > 0:
-      pc2 = nextExceptOrFinally
+    var pc = f.safePoints.pop()
-    if c.code[pc2].opcode == opcFinally:
+
-      # execute the corresponding handler, but don't quit walking the stack:
+    var matched = false
-      discard f.safePoints.pop
+    var pcEndExcept = pc
-      return (pc2+1, f)
+
-    # not the right one:
+    # Scan the chain of exceptions starting at pc.
-    discard f.safePoints.pop
+    # The structure is the following:
+    # pc - opcExcept, <end of this block>
+    #      - opcExcept, <pattern1>
+    #      - opcExcept, <pattern2>
+    #        ...
+    #      - opcExcept, <patternN>
+    #      - Exception handler body
+    #    - ... more opcExcept blocks may follow
+    #    - ... an optional opcFinally block may follow
+    #
+    # Note that the exception handler body already contains a jump to the
+    # finally block or, if that's not present, to the point where the execution
+    # should continue.
+    # Also note that opcFinally blocks are the last in the chain.
+    while c.code[pc].opcode == opcExcept:
+      # Where this Except block ends
+      pcEndExcept = pc + c.code[pc].regBx - wordExcess
+      inc pc
+
+      # A series of opcExcept follows for each exception type matched
+      while c.code[pc].opcode == opcExcept:
+        let excIndex = c.code[pc].regBx - wordExcess
+        let exceptType =
+          if excIndex > 0: c.types[excIndex].skipTypes(abstractPtrs)
+          else: nil
+
+        # echo typeToString(exceptType), " ", typeToString(raisedType)
+
+        # Determine if the exception type matches the pattern
+        if exceptType.isNil or inheritanceDiff(raisedType, exceptType) <= 0:
+          matched = true
+          break
+
+        inc pc
+
+      # Skip any further ``except`` pattern and find the first instruction of
+      # the handler body
+      while c.code[pc].opcode == opcExcept:
+        inc pc
+
+      if matched:
+        break
+
+      # If no handler in this chain is able to catch this exception we check if
+      # the "parent" chains are able to. If this chain ends with a `finally`
+      # block we must execute it before continuing.
+      pc = pcEndExcept
+
+    # Where the handler body starts
+    let pcBody = pc
+
+    if matched:
+      return (ExceptionGotoHandler, pcBody)
+    elif c.code[pc].opcode == opcFinally:
+      # The +1 here is here because we don't want to execute it since we've
+      # already pop'd this statepoint from the stack.
+      return (ExceptionGotoFinally, pc + 1)
+
+  return (ExceptionGotoUnhandled, 0)
 
 proc cleanUpOnReturn(c: PCtx; f: PStackFrame): int =
-  for s in f.safePoints:
+  # Walk up the chain of safepoints and return the PC of the first `finally`
-    var pc = s
+  # block we find or -1 if no such block is found.
+  # Note that the safepoint is removed once the function returns!
+  result = -1
+
+  # Traverse the stack starting from the end in order to execute the blocks in
+  # the inteded order
+  for i in 1 .. f.safePoints.len:
+    var pc = f.safePoints[^i]
+    # Skip the `except` blocks
     while c.code[pc].opcode == opcExcept:
-      pc = pc + c.code[pc].regBx - wordExcess
+      pc += c.code[pc].regBx - wordExcess
     if c.code[pc].opcode == opcFinally:
-      return pc
+      discard f.safePoints.pop
-  return -1
+      return pc + 1
 
 proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
   if desttyp.kind == tyString:
@@ -449,6 +486,9 @@ const
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
+  # Used to keep track of where the execution is resumed.
+  var savedPC = -1
+  var savedFrame: PStackFrame
   var regs: seq[TFullReg] # alias to tos.slots for performance
   move(regs, tos.slots)
   #echo "NEW RUN ------------------------"
@@ -456,27 +496,31 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     #{.computedGoto.}
     let instr = c.code[pc]
     let ra = instr.regA
-    #if c.traceActive:
+
     when traceCode:
       echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC
-      # message(c.config, c.debug[pc], warnUser, "Trace")
 
     case instr.opcode
     of opcEof: return regs[ra]
     of opcRet:
-      # XXX perform any cleanup actions
+      let newPc = c.cleanUpOnReturn(tos)
-      pc = tos.comesFrom
+      # Perform any cleanup action before returning
-      tos = tos.next
+      if newPc < 0:
-      let retVal = regs[0]
+        pc = tos.comesFrom
-      if tos.isNil:
+        tos = tos.next
-        #echo "RET ", retVal.rendertree
+        let retVal = regs[0]
-        return retVal
+        if tos.isNil:
+          return retVal
 
-      move(regs, tos.slots)
+        move(regs, tos.slots)
-      assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
+        assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
-      if c.code[pc].opcode == opcIndCallAsgn:
+        if c.code[pc].opcode == opcIndCallAsgn:
-        regs[c.code[pc].regA] = retVal
+          regs[c.code[pc].regA] = retVal
-        #echo "RET2 ", retVal.rendertree, " ", c.code[pc].regA
+      else:
+        savedPC = pc
+        savedFrame = tos
+        # The -1 is needed because at the end of the loop we increment `pc`
+        pc = newPc - 1
     of opcYldYoid: assert false
     of opcYldVal: assert false
     of opcAsgnInt:
@@ -1025,7 +1069,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         # it's a callback:
         c.callbacks[-prc.offset-2].value(
           VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[pointer](regs),
-                 currentException: c.currentExceptionB,
+                 currentException: c.currentExceptionA,
                  currentLineInfo: c.debug[pc]))
       elif sfImportc in prc.flags:
         if allowFFI notin c.features:
@@ -1118,44 +1162,55 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       tos.pushSafePoint(pc + rbx)
       assert c.code[pc+rbx].opcode in {opcExcept, opcFinally}
     of opcExcept:
-      # just skip it; it's followed by a jump;
+      # This opcode is never executed, it only holds informations for the
-      # we'll execute in the 'raise' handler
+      # exception handling routines.
-      let rbx = instr.regBx - wordExcess - 1 # -1 for the following 'inc pc'
+      doAssert(false)
-      inc pc, rbx
-      while c.code[pc+1].opcode == opcExcept:
-        let rbx = c.code[pc+1].regBx - wordExcess - 1
-        inc pc, rbx
-      #assert c.code[pc+1].opcode in {opcExcept, opcFinally}
-      if c.code[pc+1].opcode != opcFinally:
-        # in an except handler there is no active safe point for the 'try':
-        tos.popSafePoint()
     of opcFinally:
-      # just skip it; it's followed by the code we need to execute anyway
+      # Pop the last safepoint introduced by a opcTry. This opcode is only
+      # executed _iff_ no exception was raised in the body of the `try`
+      # statement hence the need to pop the safepoint here.
+      doAssert(savedPC < 0)
       tos.popSafePoint()
     of opcFinallyEnd:
-      if c.currentExceptionA != nil:
+      # The control flow may not resume at the next instruction since we may be
-        # we are in a cleanup run:
+      # raising an exception or performing a cleanup.
-        let (newPc, newTos) = cleanUpOnException(c, tos)
+      if not savedPC < 0:
-        if newPc-1 < 0:
+        pc = savedPC - 1
-          bailOut(c, tos)
+        savedPC = -1
-          return
+        if tos != savedFrame:
-        pc = newPc-1
+          tos = savedFrame
-        if tos != newTos:
-          tos = newTos
           move(regs, tos.slots)
     of opcRaise:
       let raised = regs[ra].node
       c.currentExceptionA = raised
       c.exceptionInstr = pc
-      let (newPc, newTos) = cleanUpOnException(c, tos)
+
-      # -1 because of the following 'inc'
+      var frame = tos
-      if newPc-1 < 0:
+      var jumpTo = findExceptionHandler(c, frame, raised)
+      while jumpTo.why == ExceptionGotoUnhandled and not frame.next.isNil:
+        frame = frame.next
+        jumpTo = findExceptionHandler(c, frame, raised)
+
+      case jumpTo.why:
+      of ExceptionGotoHandler:
+        # Jump to the handler, do nothing when the `finally` block ends.
+        savedPC = -1
+        pc = jumpTo.where - 1
+        if tos != frame:
+          tos = frame
+          move(regs, tos.slots)
+      of ExceptionGotoFinally:
+        # Jump to the `finally` block first then re-jump here to continue the
+        # traversal of the exception chain
+        savedPC = pc
+        savedFrame = tos
+        pc = jumpTo.where - 1
+        if tos != frame:
+          tos = frame
+          move(regs, tos.slots)
+      of ExceptionGotoUnhandled:
+        # Nobody handled this exception, error out.
         bailOut(c, tos)
-        return
-      pc = newPc-1
-      if tos != newTos:
-        tos = newTos
-        move(regs, tos.slots)
     of opcNew:
       ensureKind(rkNode)
       let typ = c.types[instr.regBx - wordExcess]
@@ -1295,7 +1350,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         idx = int(regs[rb+rc-1].intVal)
         callback = c.callbacks[idx].value
         args = VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[pointer](regs),
-                currentException: c.currentExceptionB,
+                currentException: c.currentExceptionA,
                 currentLineInfo: c.debug[pc])
       callback(args)
       regs[ra].node.flags.incl nfIsRef

compiler/vmgen.nim

@@ -83,9 +83,12 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
     elif opc < firstABxInstr:
       result.addf("\t$#\tr$#, r$#, r$#", opc.toStr, x.regA,
                   x.regB, x.regC)
-    elif opc in relativeJumps:
+    elif opc in relativeJumps + {opcTry}:
       result.addf("\t$#\tr$#, L$#", opc.toStr, x.regA,
                   i+x.regBx-wordExcess)
+    elif opc in {opcExcept}:
+      let idx = x.regBx-wordExcess
+      result.addf("\t$#\t$#, $#", opc.toStr, x.regA, $idx)
     elif opc in {opcLdConst, opcAsgnConst}:
       let idx = x.regBx-wordExcess
       result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA,
@@ -480,10 +483,13 @@ proc genType(c: PCtx; typ: PType): int =
 proc genTry(c: PCtx; n: PNode; dest: var TDest) =
   if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ)
   var endings: seq[TPosition] = @[]
-  let elsePos = c.xjmp(n, opcTry, 0)
+  let ehPos = c.xjmp(n, opcTry, 0)
   c.gen(n.sons[0], dest)
   c.clearDest(n, dest)
-  c.patch(elsePos)
+  # Add a jump past the exception handling code
+  endings.add(c.xjmp(n, opcJmp, 0))
+  # This signals where the body ends and where the exception handling begins
+  c.patch(ehPos)
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.kind != nkFinally:
@@ -499,14 +505,14 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
         c.gABx(it, opcExcept, 0, 0)
       c.gen(it.lastSon, dest)
       c.clearDest(n, dest)
-      if i < sonsLen(n)-1:
+      if i < sonsLen(n):
         endings.add(c.xjmp(it, opcJmp, 0))
       c.patch(endExcept)
-  for endPos in endings: c.patch(endPos)
   let fin = lastSon(n)
   # we always generate an 'opcFinally' as that pops the safepoint
-  # from the stack
+  # from the stack if no exception is raised in the body.
   c.gABx(fin, opcFinally, 0, 0)
+  for endPos in endings: c.patch(endPos)
   if fin.kind == nkFinally:
     c.gen(fin.sons[0])
     c.clearDest(n, dest)
@@ -2214,9 +2220,9 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "calc":
+    # if s.name.s == "fun1":
-    #  echo renderTree(body)
+    #   echo renderTree(body)
-    #  c.echoCode(result)
+    #   c.echoCode(result)
     c.prc = oldPrc
   else:
     c.prc.maxSlots = s.offset

tests/exception/texceptions.nim

@@ -74,3 +74,54 @@ block: #10417
       moo()
 
   doAssert(bar == 1)
+
+# Make sure the VM handles the exceptions correctly
+block:
+  proc fun1(): seq[int] =
+    try:
+      try:
+        raise newException(ValueError, "xx")
+      except:
+        doAssert("xx" == getCurrentExceptionMsg())
+        raise newException(KeyError, "yy")
+    except:
+      doAssert("yy" == getCurrentExceptionMsg())
+      result.add(1212)
+    try:
+      try:
+        raise newException(AssertionError, "a")
+      finally:
+        result.add(42)
+    except AssertionError:
+      result.add(99)
+    finally:
+      result.add(10)
+    result.add(4)
+    result.add(0)
+    try:
+      result.add(1)
+    except KeyError:
+      result.add(-1)
+    except ValueError:
+      result.add(-1)
+    except IndexError:
+      result.add(2)
+    except:
+      result.add(3)
+
+    try:
+      try:
+        result.add(1)
+        return
+      except:
+        result.add(-1)
+      finally:
+        result.add(2)
+    except KeyError:
+      doAssert(false)
+    finally:
+      result.add(3)
+
+  let x1 = fun1()
+  const x2 = fun1()
+  doAssert(x1 == x2)

tests/vm/tmisc_vm.nim

@@ -49,3 +49,23 @@ static:
     echo "caught Defect"
   except ValueError:
     echo "caught ValueError"
+
+# bug #10538
+
+block:
+  proc fun1(): seq[int] =
+    try:
+      try:
+        result.add(1)
+        return
+      except:
+        result.add(-1)
+      finally:
+        result.add(2)
+    finally:
+      result.add(3)
+    result.add(4)
+
+  let x1 = fun1()
+  const x2 = fun1()
+  doAssert(x1 == x2)