better tester; yet another iterator bugfix

doc/pegdocs.txt

@@ -97,6 +97,10 @@ macro              meaning
 ``\n``             any newline combination: ``\10 / \13\10 / \13``
 ``\i``             ignore case for matching; use this at the start of the PEG
 ``\y``             ignore style for matching; use this at the start of the PEG
+``\skip`` pat      skip pattern *pat* before trying to match other tokens; 
+                   this is useful for whitespace skipping, for example:
+                   ``\skip(\s*) {\ident} ':' {\ident}`` matches key value 
+                   pairs ignoring whitespace around the ``':'``.
 ``\ident``         a standard ASCII identifier: ``[a-zA-Z_][a-zA-Z_0-9]*``
 ``\letter``        any Unicode letter
 ``\upper``         any Unicode uppercase letter
@@ -198,8 +202,8 @@ As a regular expression ``\[.*\]`` matches the longest possible text between
 deterministic: ``.*`` consumes the rest of the input, so ``\]`` never matches.
 As a PEG this needs to be written as: ``\[ ( !\] . )* \]`` (or ``\[ @ \]``).
 
-Note that the regular expression does not behave as intended either:
+Note that the regular expression does not behave as intended either: in the
-``*`` should not be greedy, so ``\[.*?\]`` should be used.
+example ``*`` should not be greedy, so ``\[.*?\]`` should be used instead.
 
 
 PEG construction

lib/pure/pegs.nim

@@ -1351,6 +1351,7 @@ type
     modifier: TModifier
     captures: int
     identIsVerbatim: bool
+    skip: TPeg
 
 proc pegError(p: TPegParser, msg: string, line = -1, col = -1) =
   var e: ref EInvalidPeg
@@ -1388,6 +1389,30 @@ proc modifiedBackref(s: int, m: TModifier): TPeg =
   of modIgnoreCase: result = backRefIgnoreCase(s)
   of modIgnoreStyle: result = backRefIgnoreStyle(s)
 
+proc builtin(p: var TPegParser): TPeg =
+  # do not use "y", "skip" or "i" as these would be ambiguous
+  case p.tok.literal
+  of "n": result = newLine()
+  of "d": result = charset({'0'..'9'})
+  of "D": result = charset({'\1'..'\xff'} - {'0'..'9'})
+  of "s": result = charset({' ', '\9'..'\13'})
+  of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'})
+  of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'})
+  of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'})
+  of "a": result = charset({'a'..'z', 'A'..'Z'})
+  of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'})
+  of "ident": result = pegs.ident
+  of "letter": result = UnicodeLetter()
+  of "upper": result = UnicodeUpper()
+  of "lower": result = UnicodeLower()
+  of "title": result = UnicodeTitle()
+  of "white": result = UnicodeWhitespace()
+  else: pegError(p, "unknown built-in: " & p.tok.literal)
+
+proc token(terminal: TPeg, p: TPegParser): TPeg = 
+  if p.skip.kind == pkEmpty: result = terminal
+  else: result = sequence(p.skip, terminal)
+
 proc primary(p: var TPegParser): TPeg =
   case p.tok.kind
   of tkAmp:
@@ -1401,31 +1426,31 @@ proc primary(p: var TPegParser): TPeg =
     return @primary(p)
   of tkCurlyAt:
     getTok(p)
-    return @@primary(p)
+    return @@primary(p).token(p)
   else: nil
   case p.tok.kind
   of tkIdentifier:
     if p.identIsVerbatim: 
       var m = p.tok.modifier
       if m == modNone: m = p.modifier
-      result = modifiedTerm(p.tok.literal, m)
+      result = modifiedTerm(p.tok.literal, m).token(p)
       getTok(p)
     elif not arrowIsNextTok(p):
       var nt = getNonTerminal(p, p.tok.literal)
       incl(nt.flags, ntUsed)
-      result = nonTerminal(nt)
+      result = nonTerminal(nt).token(p)
       getTok(p)
     else:
       pegError(p, "expression expected, but found: " & p.tok.literal)
   of tkStringLit:
     var m = p.tok.modifier
     if m == modNone: m = p.modifier
-    result = modifiedTerm(p.tok.literal, m)
+    result = modifiedTerm(p.tok.literal, m).token(p)
     getTok(p)
   of tkCharSet:
     if '\0' in p.tok.charset:
       pegError(p, "binary zero ('\\0') not allowed in character class")
-    result = charset(p.tok.charset)
+    result = charset(p.tok.charset).token(p)
     getTok(p)
   of tkParLe:
     getTok(p)
@@ -1433,41 +1458,25 @@ proc primary(p: var TPegParser): TPeg =
     eat(p, tkParRi)
   of tkCurlyLe:
     getTok(p)
-    result = capture(parseExpr(p))
+    result = capture(parseExpr(p)).token(p)
     eat(p, tkCurlyRi)
     inc(p.captures)
   of tkAny:
-    result = any()
+    result = any().token(p)
     getTok(p)
   of tkAnyRune:
-    result = anyRune()
+    result = anyRune().token(p)
     getTok(p)
   of tkBuiltin:
-    case p.tok.literal
+    result = builtin(p).token(p)
-    of "n": result = newLine()
-    of "d": result = charset({'0'..'9'})
-    of "D": result = charset({'\1'..'\xff'} - {'0'..'9'})
-    of "s": result = charset({' ', '\9'..'\13'})
-    of "S": result = charset({'\1'..'\xff'} - {' ', '\9'..'\13'})
-    of "w": result = charset({'a'..'z', 'A'..'Z', '_', '0'..'9'})
-    of "W": result = charset({'\1'..'\xff'} - {'a'..'z','A'..'Z','_','0'..'9'})
-    of "a": result = charset({'a'..'z', 'A'..'Z'})
-    of "A": result = charset({'\1'..'\xff'} - {'a'..'z', 'A'..'Z'})
-    of "ident": result = pegs.ident
-    of "letter": result = UnicodeLetter()
-    of "upper": result = UnicodeUpper()
-    of "lower": result = UnicodeLower()
-    of "title": result = UnicodeTitle()
-    of "white": result = UnicodeWhitespace()
-    else: pegError(p, "unknown built-in: " & p.tok.literal)
     getTok(p)
   of tkEscaped:
-    result = term(p.tok.literal[0])
+    result = term(p.tok.literal[0]).token(p)
     getTok(p)
   of tkDollar:
     var m = p.tok.modifier
     if m == modNone: m = p.modifier
-    result = modifiedBackRef(p.tok.index, m)
+    result = modifiedBackRef(p.tok.index, m).token(p)
     if p.tok.index < 0 or p.tok.index > p.captures: 
       pegError(p, "invalid back reference index: " & $p.tok.index)
     getTok(p)
@@ -1522,7 +1531,7 @@ proc parseRule(p: var TPegParser): PNonTerminal =
   
 proc rawParse(p: var TPegParser): TPeg =
   ## parses a rule or a PEG expression
-  if p.tok.kind == tkBuiltin:
+  while p.tok.kind == tkBuiltin:
     case p.tok.literal
     of "i":
       p.modifier = modIgnoreCase
@@ -1530,7 +1539,10 @@ proc rawParse(p: var TPegParser): TPeg =
     of "y":
       p.modifier = modIgnoreStyle
       getTok(p)
-    else: nil
+    of "skip":
+      getTok(p)
+      p.skip = ?primary(p)
+    else: break
   if p.tok.kind == tkIdentifier and arrowIsNextTok(p):
     result = parseRule(p).rule
     while p.tok.kind != tkEof:
@@ -1676,4 +1688,7 @@ when isMainModule:
   assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
   assert(not match("456678", peg"(\letter)+"))
 
+  assert("var1 = key; var2 = key2".replace(
+    peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
+         "var1<-keykey;var2<-key2key2")
 

rod/transf.nim

@@ -188,6 +188,41 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
 proc transformSym(c: PTransf, n: PNode): PTransNode = 
   result = PTransNode(transformSymAux(c, n))
 
+proc transformVarSection(c: PTransf, v: PNode): PTransNode =
+  result = newTransNode(v)
+  for i in countup(0, sonsLen(v)-1): 
+    var it = v.sons[i]
+    if it.kind == nkCommentStmt: 
+      result[i] = PTransNode(it)
+    elif it.kind == nkIdentDefs: 
+      if (it.sons[0].kind != nkSym):
+        InternalError(it.info, "transformVarSection")
+      var newVar = copySym(it.sons[0].sym)
+      incl(newVar.flags, sfFromGeneric) 
+      # fixes a strange bug for rodgen:
+      #include(it.sons[0].sym.flags, sfFromGeneric);
+      newVar.owner = getCurrOwner(c)
+      IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
+      var defs = newTransNode(nkIdentDefs, it.info, 3)
+      defs[0] = newSymNode(newVar).PTransNode
+      defs[1] = it.sons[1].PTransNode
+      defs[2] = transform(c, it.sons[2])
+      result[i] = defs
+    else: 
+      if it.kind != nkVarTuple: 
+        InternalError(it.info, "transformVarSection: not nkVarTuple")
+      var L = sonsLen(it)
+      var defs = newTransNode(it.kind, it.info, L)
+      for j in countup(0, L-3): 
+        var newVar = copySym(it.sons[j].sym)
+        incl(newVar.flags, sfFromGeneric)
+        newVar.owner = getCurrOwner(c)
+        IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
+        defs[j] = newSymNode(newVar).PTransNode
+      assert(it.sons[L-2] == nil)
+      defs[L-1] = transform(c, it.sons[L-1])
+      result[i] = defs
+
 proc hasContinue(n: PNode): bool = 
   if n == nil: return 
   case n.kind
@@ -235,6 +270,21 @@ proc unpackTuple(c: PTransf, n: PNode, father: PTransNode) =
     add(father, newAsgnStmt(c, c.transCon.forStmt.sons[i], 
         transform(c, newTupleAccess(n, i))))
 
+proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode = 
+  if n == nil: return
+  case n.kind
+  of nkSym: 
+    return transformSym(c, n)
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: 
+    # nothing to be done for leaves:
+    result = PTransNode(n)
+  of nkVarSection:
+    result = transformVarSection(c, n)
+  else:
+    result = newTransNode(n)
+    for i in countup(0, sonsLen(n)-1): 
+      result[i] =  introduceNewLocalVars(c, n.sons[i])
+
 proc transformYield(c: PTransf, n: PNode): PTransNode = 
   result = newTransNode(nkStmtList, n.info, 0)
   var e = n.sons[0]
@@ -255,43 +305,8 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
     # common case
     add(result, c.transCon.forLoopBody)
   else: 
-    # we need to transform again to introduce new local variables:
+    # we need to introduce new local variables:
-    add(result, transform(c, c.transCon.forLoopBody.pnode))
+    add(result, introduceNewLocalVars(c, c.transCon.forLoopBody.pnode))
-
-proc transformVarSection(c: PTransf, v: PNode): PTransNode =
-  result = newTransNode(v)
-  for i in countup(0, sonsLen(v)-1): 
-    var it = v.sons[i]
-    if it.kind == nkCommentStmt: 
-      result[i] = PTransNode(it)
-    elif it.kind == nkIdentDefs: 
-      if (it.sons[0].kind != nkSym):
-        InternalError(it.info, "transformVarSection")
-      var newVar = copySym(it.sons[0].sym)
-      incl(newVar.flags, sfFromGeneric) 
-      # fixes a strange bug for rodgen:
-      #include(it.sons[0].sym.flags, sfFromGeneric);
-      newVar.owner = getCurrOwner(c)
-      IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
-      var defs = newTransNode(nkIdentDefs, it.info, 3)
-      defs[0] = newSymNode(newVar).PTransNode
-      defs[1] = it.sons[1].PTransNode
-      defs[2] = transform(c, it.sons[2])
-      result[i] = defs
-    else: 
-      if it.kind != nkVarTuple: 
-        InternalError(it.info, "transformVarSection: not nkVarTuple")
-      var L = sonsLen(it)
-      var defs = newTransNode(it.kind, it.info, L)
-      for j in countup(0, L-3): 
-        var newVar = copySym(it.sons[j].sym)
-        incl(newVar.flags, sfFromGeneric)
-        newVar.owner = getCurrOwner(c)
-        IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
-        defs[j] = newSymNode(newVar).PTransNode
-      assert(it.sons[L-2] == nil)
-      defs[L-1] = transform(c, it.sons[L-1])
-      result[i] = defs
 
 proc addVar(father, v: PNode) = 
   var vpart = newNodeI(nkIdentDefs, v.info)

tests/tester.nim

@@ -92,6 +92,9 @@ proc initResults: TResults =
   result.passed = 0
   result.data = ""
 
+proc `$`(x: TResults): string = 
+  result = "Tests passed: " & $x.passed & "/" & $x.total & "<br />\n"
+
 proc colorBool(b: bool): string =
   if b: result = "<span style=\"color:green\">yes</span>" 
   else: result = "<span style=\"color:red\">no</span>"
@@ -116,10 +119,13 @@ proc addResult(r: var TResults, test, given: string,
 proc listResults(reject, compile, run: TResults) =
   var s = "<html>"
   s.add("<h1>Tests to Reject</h1>\n")
+  s.add($reject)
   s.add(TableHeader4 & reject.data & TableFooter)
   s.add("<br /><br /><br /><h1>Tests to Compile</h1>\n")
+  s.add($compile)
   s.add(TableHeader3 & compile.data & TableFooter)
   s.add("<br /><br /><br /><h1>Tests to Run</h1>\n")
+  s.add($run)
   s.add(TableHeader4 & run.data & TableFooter)
   s.add("</html>")
   var outp: TFile
@@ -145,6 +151,8 @@ proc reject(r: var TResults, dir, options: string) =
   
   for test in os.walkFiles(dir / "t*.nim"):
     var t = extractFilename(test)
+    inc(r.total)
+    echo t
     var expected = findSpec(specs, t)
     var given = callCompiler(test, options)
     cmpMsgs(r, specs[expected], given, t)
@@ -153,6 +161,7 @@ proc compile(r: var TResults, pattern, options: string) =
   for test in os.walkFiles(pattern): 
     var t = extractFilename(test)
     inc(r.total)
+    echo t
     var given = callCompiler(test, options)
     r.addResult(t, given.msg, not given.err)
     if not given.err: inc(r.passed)
@@ -162,6 +171,7 @@ proc run(r: var TResults, dir, options: string) =
   for test in os.walkFiles(dir / "t*.nim"): 
     var t = extractFilename(test)
     inc(r.total)
+    echo t
     var given = callCompiler(test, options)
     if given.err:
       r.addResult(t, "", given.msg, not given.err)

todo.txt

@@ -1,4 +1,3 @@
-- Tester needs to count failures
 - we need a way to disable tests
 
 

web/news.txt

@@ -2,6 +2,7 @@
 News
 ====
 
+
 2010-XX-XX Version 0.8.12 released
 ==================================
 
@@ -29,6 +30,7 @@ Additions
 - Pegs support a *captured search loop operator* ``{@}``.
 - Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``,
   ``\title``, ``\white``.
+- Pegs support the new built-in ``\skip`` operation.
 
 
 2010-10-20 Version 0.8.10 released