better html generator for the tester; fixes some VM bugs

compiler/sem.nim

@@ -15,7 +15,7 @@ import
   magicsys, parser, nversion, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
   semthreads, intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
-  evaltempl, patterns, parampatterns, sempass2, pretty
+  evaltempl, patterns, parampatterns, sempass2, pretty, semmacrosanity
 
 # implementation
 
@@ -186,6 +186,13 @@ when false:
 proc fixupTypeAfterEval(c: PContext, evaluated, eOrig: PNode): PNode =
   # recompute the types as 'eval' isn't guaranteed to construct types nor
   # that the types are sound:
+  when true:
+    if eOrig.typ.kind in {tyExpr, tyStmt, tyTypeDesc}:
+      result = semExprWithType(c, evaluated)
+    else:
+      result = evaluated
+      semmacrosanity.annotateType(result, eOrig.typ)
+  else:
     result = semExprWithType(c, evaluated)
     #result = fitNode(c, e.typ, result) inlined with special case:
     let arg = result

compiler/semexprs.nim

@@ -1068,7 +1068,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     else: 
       localError(n.info, errIndexTypesDoNotMatch)
     result = n
-  else: nil
+  else: discard
   
 proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = 
   result = semSubscript(c, n, flags)

compiler/semmacrosanity.nim

@@ -0,0 +1,89 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements type sanity checking for ASTs resulting from macros. Lots of
+## room for improvement here.
+
+import ast, astalgo, msgs, types
+
+proc ithField(n: PNode, field: int): PSym =
+  result = nil
+  case n.kind
+  of nkRecList:
+    for i in countup(0, sonsLen(n) - 1): 
+      result = ithField(n.sons[i], field-i)
+      if result != nil: return 
+  of nkRecCase:
+    if n.sons[0].kind != nkSym: internalError(n.info, "ithField")
+    result = ithField(n.sons[0], field-1)
+    if result != nil: return
+    for i in countup(1, sonsLen(n) - 1):
+      case n.sons[i].kind
+      of nkOfBranch, nkElse:
+        result = ithField(lastSon(n.sons[i]), field-1)
+        if result != nil: return
+      else: internalError(n.info, "ithField(record case branch)")
+  of nkSym:
+    if field == 0: result = n.sym
+  else: discard
+
+proc annotateType*(n: PNode, t: PType) =
+  let x = t.skipTypes(abstractInst)
+  # Note: x can be unequal to t and we need to be careful to use 't'
+  # to not to skip tyGenericInst
+  case n.kind
+  of nkPar:
+    if x.kind == tyObject:
+      n.typ = t
+      for i in 0 .. <n.len:
+        let field = x.n.ithField(i)
+        if field.isNil: globalError n.info, "invalid " & $i & "th field"
+        else: annotateType(n.sons[i], field.typ)
+    elif x.kind == tyTuple:
+      n.typ = t
+      for i in 0 .. <n.len:
+        if i >= x.len: globalError n.info, "invalid " & $i & "th field"
+        else: annotateType(n.sons[i], x.sons[i])
+    elif x.kind == tyProc and x.callConv == ccClosure:
+      n.typ = t
+    else:
+      globalError(n.info, "() must have an object or tuple type")
+  of nkBracket:
+    if x.kind in {tyArrayConstr, tyArray, tySequence, tyOpenarray}:
+      n.typ = t
+      for m in n: annotateType(m, x.elemType)
+    else:
+      globalError(n.info, "[] must have some form of array type")
+  of nkCurly:
+    if x.kind in {tySet}:
+      n.typ = t
+      for m in n: annotateType(m, x.elemType)
+    else:
+      globalError(n.info, "{} must have the set type")
+  of nkFloatLit..nkFloat128Lit:
+    if x.kind in {tyFloat..tyFloat128}:
+      n.typ = t
+    else:
+      globalError(n.info, "float literal must have some float type")
+  of nkCharLit..nkUInt64Lit:
+    if x.kind in {tyInt..tyUInt64, tyBool, tyChar, tyEnum}:
+      n.typ = t
+    else:
+      globalError(n.info, "integer literal must have some int type")
+  of nkStrLit..nkTripleStrLit:
+    if x.kind in {tyString, tyCString}:
+      n.typ = t
+    else:
+      globalError(n.info, "string literal must be of some string type")    
+  of nkNilLit:
+    if x.kind in NilableTypes:
+      n.typ = t
+    else:
+      globalError(n.info, "nil literal must be of some pointer type")
+  else: discard

compiler/vm.nim

@@ -172,7 +172,9 @@ proc asgnComplex(x, y: PNode) =
   else:
     if x.kind notin {nkEmpty..nkNilLit}:
       let y = y.copyValue
-      for i in countup(0, sonsLen(y) - 1): addSon(x, y.sons[i])
+      for i in countup(0, sonsLen(y) - 1): 
+        if i < x.len: x.sons[i] = y.sons[i]
+        else: addSon(x, y.sons[i])
 
 template getstr(a: expr): expr =
   (if a.kind in {nkStrLit..nkTripleStrLit}: a.strVal else: $chr(int(a.intVal)))
@@ -319,6 +321,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       decodeB(nkIntLit)
       regs[ra].intVal = regs[rb].intVal
     of opcAsgnStr:
+      if regs[instr.regB].kind == nkNilLit:
+        decodeB(nkNilLit)
+      else:
         decodeB(nkStrLit)
         regs[ra].strVal = regs[rb].strVal
     of opcAsgnFloat:
@@ -336,6 +341,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       # a = b[c]
       let rb = instr.regB
       let rc = instr.regC
+      if regs[rc].intVal > high(int):
+        stackTrace(c, tos, pc, errIndexOutOfBounds)
       let idx = regs[rc].intVal.int
       # XXX what if the array is not 0-based? -> codegen should insert a sub
       assert regs[rb].kind != nkMetaNode
@@ -373,7 +380,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       let rb = instr.regB
       let rc = instr.regC
       # XXX this creates a wrong alias
-      #Message(c.debug[pc], warnUser, $regs[rb].len & " " & $rc)
+      #Message(c.debug[pc], warnUser, $regs[rb].safeLen & " " & $rc)
       asgnComplex(regs[ra], regs[rb].sons[rc])
     of opcWrObj:
       # a.b = c
@@ -427,6 +434,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       regs[ra].intVal = regs[rb].skipMeta.len - imm
     of opcLenStr:
       decodeBImm(nkIntLit)
+      if regs[rb].kind == nkNilLit:
+        stackTrace(c, tos, pc, errNilAccess)
+      else:
         assert regs[rb].kind in {nkStrLit..nkTripleStrLit}
         regs[ra].intVal = regs[rb].strVal.len - imm
     of opcIncl:
@@ -738,11 +748,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       inc pc
       ensureKind(nkBracket)
       let instr2 = c.code[pc]
-      let rb = instr2.regA
+      let count = regs[instr2.regA].intVal.int
       regs[ra].typ = typ
-      newSeq(regs[ra].sons, rb)
+      newSeq(regs[ra].sons, count)
-      for i in 0 .. <rb:
+      for i in 0 .. <count:
-        regs[ra].sons[i] = getNullValue(typ, regs[ra].info)
+        regs[ra].sons[i] = getNullValue(typ.sons[0], regs[ra].info)
     of opcNewStr:
       decodeB(nkStrLit)
       regs[ra].strVal = newString(regs[rb].intVal.int)
@@ -1044,7 +1054,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
     inc pc
 
 proc fixType(result, n: PNode) {.inline.} =
-  # XXX do it deeply for complex values
+  # XXX do it deeply for complex values; there seems to be no simple
+  # solution except to check it deeply here.
   #if result.typ.isNil: result.typ = n.typ
 
 proc execute(c: PCtx, start: int): PNode =

compiler/vmgen.nim

@@ -444,21 +444,43 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) =
   c.freeTempRange(x, n.len)
   clearDest(n, dest)
 
+proc needsAsgnPatch(n: PNode): bool = 
+  n.kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}
+
+proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
+  case le.kind
+  of nkBracketExpr:
+    let dest = c.genx(le.sons[0])
+    let idx = c.genx(le.sons[1])
+    c.gABC(le, opcWrArrRef, dest, idx, value)
+  of nkDotExpr, nkCheckedFieldExpr:
+    # XXX field checks here
+    let left = if le.kind == nkDotExpr: le else: le.sons[0]
+    let dest = c.genx(left.sons[0])
+    let idx = c.genx(left.sons[1])
+    c.gABC(left, opcWrObjRef, dest, idx, value)
+  else:
+    discard
+
 proc genNew(c: PCtx; n: PNode) =
-  let dest = c.genx(n.sons[1])
+  let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ)
+             else: c.genx(n.sons[1])
   # we use the ref's base type here as the VM conflates 'ref object' 
   # and 'object' since internally we already have a pointer.
   c.gABx(n, opcNew, dest, 
          c.genType(n.sons[1].typ.skipTypes(abstractVar).sons[0]))
+  c.genAsgnPatch(n.sons[1], dest)
   c.freeTemp(dest)
 
 proc genNewSeq(c: PCtx; n: PNode) =
-  let dest = c.genx(n.sons[1])
+  let dest = if needsAsgnPatch(n.sons[1]): c.getTemp(n.sons[1].typ)
+             else: c.genx(n.sons[1])
   c.gABx(n, opcNewSeq, dest, c.genType(n.sons[1].typ.skipTypes(abstractVar)))
   let tmp = c.genx(n.sons[2])
   c.gABx(n, opcNewSeq, tmp, 0)
-  c.freeTemp(dest)
   c.freeTemp(tmp)
+  c.genAsgnPatch(n.sons[1], dest)
+  c.freeTemp(dest)
 
 proc genUnaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   let tmp = c.genx(n.sons[1])
@@ -1428,7 +1450,7 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "importImpl_forward" or s.name.s == "importImpl":
+    #if s.name.s == "rawGet":
     #  c.echoCode(result)
     #  echo renderTree(body)
     c.prc = oldPrc

lib/pure/collections/tables.nim

@@ -153,7 +153,7 @@ proc add*[A, B](t: var TTable[A, B], key: A, val: B) =
   
 proc del*[A, B](t: var TTable[A, B], key: A) =
   ## deletes `key` from hash table `t`.
-  var index = rawGet(t, key)
+  let index = rawGet(t, key)
   if index >= 0:
     t.data[index].slot = seDeleted
     dec(t.counter)

tests/testament/backend.nim

@@ -77,7 +77,8 @@ proc getMachine*: MachineId =
                          name, system.hostOS, system.hostCPU).MachineId
 
 proc getCommit: CommitId =
-  let hash = "git log -n 1"()
+  const commLen = "commit ".len
+  let hash = "git log -n 1"()[commLen..commLen+10]
   let branch = "git symbolic-ref --short HEAD"()
   if hash.len == 0 or branch.len == 0: quit "cannot determine git HEAD"
   

tests/testament/categories.nim

@@ -221,6 +221,14 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
 
 const AdditionalCategories = ["debugger", "tools", "examples", "stdlib"]
 
+proc `&.?`(a, b: string): string =
+  # candidate for the stdlib?
+  result = if b.startswith(a): b else: a & b
+
+proc `&?.`(a, b: string): string =
+  # candidate for the stdlib?
+  result = if a.endswith(b): a else: a & b
+
 proc processCategory(r: var TResults, cat: Category, options: string) =
   case cat.string.normalize
   of "rodfiles":
@@ -252,5 +260,5 @@ proc processCategory(r: var TResults, cat: Category, options: string) =
     compileExample(r, "examples/gtk/*.nim", options, cat)
     compileExample(r, "examples/talk/*.nim", options, cat)
   else:
-    for name in os.walkFiles(cat.string / "t*.nim"):
+    for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"):
       testSpec r, makeTest(name, options, cat)

tests/testament/htmlgen.nim

@@ -47,39 +47,57 @@ div.tabContent.hide { display: none; }
     function init() {
       // Grab the tab links and content divs from the page
       var tabListItems = document.getElementById('tabs').childNodes;
-      for ( var i = 0; i < tabListItems.length; i++ ) {
+      for (var i = 0; i < tabListItems.length; i++) {
-        if ( tabListItems[i].nodeName == "LI" ) {
+        if (tabListItems[i].nodeName == "LI") {
-          var tabLink = getFirstChildWithTagName( tabListItems[i], 'A' );
+          var tabLink = getFirstChildWithTagName(tabListItems[i], 'A');
-          var id = getHash( tabLink.getAttribute('href') );
+          var id = getHash(tabLink.getAttribute('href'));
           tabLinks[id] = tabLink;
-          contentDivs[id] = document.getElementById( id );
+          contentDivs[id] = document.getElementById(id);
         }
       }
       // Assign onclick events to the tab links, and
       // highlight the first tab
       var i = 0;
-      for ( var id in tabLinks ) {
+      for (var id in tabLinks) {
         tabLinks[id].onclick = showTab;
         tabLinks[id].onfocus = function() { this.blur() };
-        if ( i == 0 ) tabLinks[id].className = 'selected';
+        if (i == 0) tabLinks[id].className = 'selected';
         i++;
       }
       // Hide all content divs except the first
       var i = 0;
-      for ( var id in contentDivs ) {
+      for (var id in contentDivs) {
-        if ( i != 0 ) contentDivs[id].className = 'tabContent hide';
+        if (i != 0) contentDivs[id].className = 'tabContent hide';
         i++;
       }
     }
 
-    function getFirstChildWithTagName( element, tagName ) {
+    function showTab() {
-      for ( var i = 0; i < element.childNodes.length; i++ ) {
+      var selectedId = getHash(this.getAttribute('href'));
-        if ( element.childNodes[i].nodeName == tagName ) return element.childNodes[i];
+
+      // Highlight the selected tab, and dim all others.
+      // Also show the selected content div, and hide all others.
+      for (var id in contentDivs) {
+        if (id == selectedId) {
+          tabLinks[id].className = 'selected';
+          contentDivs[id].className = 'tabContent';
+        } else {
+          tabLinks[id].className = '';
+          contentDivs[id].className = 'tabContent hide';
         }
       }
-    function getHash( url ) {
+      // Stop the browser following the link
-      var hashPos = url.lastIndexOf ( '#' );
+      return false;
-      return url.substring( hashPos + 1 );
+    }
+
+    function getFirstChildWithTagName(element, tagName) {
+      for (var i = 0; i < element.childNodes.length; i++) {
+        if (element.childNodes[i].nodeName == tagName) return element.childNodes[i];
+      }
+    }
+    function getHash(url) {
+      var hashPos = url.lastIndexOf('#');
+      return url.substring(hashPos + 1);
     }
     </script>
 

tests/testament/tester.nim

@@ -245,7 +245,7 @@ proc main() =
   case action
   of "all":
     for kind, dir in walkDir("tests"):
-      if kind == pcDir and dir != "testament":
+      if kind == pcDir and dir notin ["testament", "testdata", "nimcache"]:
         processCategory(r, Category(dir), p.cmdLineRest.string)
     for a in AdditionalCategories:
       processCategory(r, Category(a), p.cmdLineRest.string)