next steps towards term rewriting macros; simple examples work

compiler/ast.nim

@@ -176,7 +176,7 @@ type
     nkFromStmt,           # a from * import statement
     nkIncludeStmt,        # an include statement
     nkBindStmt,           # a bind statement
-    nkPatternStmt,        # a pattern statement ('as' statement)
+    nkPattern,            # a pattern statement ('as' statement)
     nkCommentStmt,        # a comment statement
     nkStmtListExpr,       # a statement list followed by an expr; this is used
                           # to allow powerful multi-line templates
@@ -687,13 +687,14 @@ const
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfAllConst}
   namePos* = 0
-  genericParamsPos* = 1
+  patternPos* = 1    # empty except for term rewriting macros
-  paramsPos* = 2
+  genericParamsPos* = 2
-  pragmasPos* = 3
+  paramsPos* = 3
-  patternPos* = 4    # empty except for term rewriting macros
+  pragmasPos* = 4
-  bodyPos* = 5       # position of body; use rodread.getBody() instead!
+  exceptionPos* = 5  # will be used for exception tracking
-  resultPos* = 6
+  bodyPos* = 6       # position of body; use rodread.getBody() instead!
-  dispatcherPos* = 7 # caution: if method has no 'result' it can be position 5!
+  resultPos* = 7
+  dispatcherPos* = 8 # caution: if method has no 'result' it can be position 5!
 
   nkCallKinds* = {nkCall, nkInfix, nkPrefix, nkPostfix,
                   nkCommand, nkCallStrLit}

compiler/commands.nim

@@ -184,6 +184,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "taintmode": result = contains(gGlobalOptions, optTaintMode)
   of "tlsemulation": result = contains(gGlobalOptions, optTlsEmulation)
   of "implicitstatic": result = contains(gOptions, optImplicitStatic)
+  of "patterns": result = contains(gOptions, optPatterns)
   else: InvalidCmdLineOption(passCmd1, switch, info)
   
 proc processPath(path: string): string = 
@@ -315,6 +316,8 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "taintmode": ProcessOnOffSwitchG({optTaintMode}, arg, pass, info)
   of "implicitstatic":
     ProcessOnOffSwitch({optImplicitStatic}, arg, pass, info)
+  of "patterns":
+    ProcessOnOffSwitch({optPatterns}, arg, pass, info)
   of "opt":
     expectArg(switch, arg, pass, info)
     case arg.normalize

compiler/options.nim

@@ -26,8 +26,9 @@ type                          # please make sure we have under 32 options
     optByRef,                 # use pass by ref for objects
                               # (for interfacing with C)
     optProfiler,              # profiler turned on
-    optImplicitStatic         # optimization: implicit at compile time
+    optImplicitStatic,        # optimization: implicit at compile time
                               # evaluation
+    optPatterns               # en/disable pattern matching
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake, optBoehmGC, optRefcGC, optDeadCodeElim, 
@@ -82,7 +83,8 @@ const
 var 
   gOptions*: TOptions = {optObjCheck, optFieldCheck, optRangeCheck, 
                          optBoundsCheck, optOverflowCheck, optAssert, optWarns, 
-                         optHints, optStackTrace, optLineTrace}
+                         optHints, optStackTrace, optLineTrace,
+                         optPatterns}
   gGlobalOptions*: TGlobalOptions = {optRefcGC, optThreadAnalysis}
   gExitcode*: int8
   searchPaths*: TLinkedList

compiler/parser.nim

@@ -700,10 +700,12 @@ proc parseDoBlock(p: var TParser): PNode =
   eat(p, tkColon)
   result = newNodeI(nkDo, info)
   addSon(result, ast.emptyNode)       # no name part
+  addSon(result, ast.emptyNode)       # no pattern part
   addSon(result, ast.emptyNode)       # no generic parameters
   addSon(result, params)
   addSon(result, pragmas)
   skipComment(p, result)
+  addSon(result, ast.emptyNode)       # no exception list
   addSon(result, parseStmt(p))
 
 proc parseDoBlocks(p: var TParser, call: PNode) =
@@ -720,14 +722,16 @@ proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
   let hasSignature = p.tok.tokType in {tkParLe, tkColon}
   params = parseParamList(p)
   pragmas = optPragmas(p)
-  if (p.tok.tokType == tkEquals) and isExpr: 
+  if p.tok.tokType == tkEquals and isExpr: 
     result = newNodeI(nkLambda, info)
     addSon(result, ast.emptyNode)       # no name part
+    addSon(result, ast.emptyNode)       # no pattern
     addSon(result, ast.emptyNode)       # no generic parameters
     addSon(result, params)
     addSon(result, pragmas)
     getTok(p)
     skipComment(p, result)
+    addSon(result, ast.emptyNode)       # no exception list
     addSon(result, parseStmt(p))
   else: 
     result = newNodeI(nkProcTy, info)
@@ -961,23 +965,14 @@ proc parseYieldOrDiscard(p: var TParser, kind: TNodeKind): PNode =
   optInd(p, result)
   addSon(result, parseExpr(p))
 
-proc parseBreakOrContinue(p: var TParser, kind: TNodeKind): PNode = 
+proc parseBreakOrContinue(p: var TParser, kind: TNodeKind): PNode =
   result = newNodeP(kind, p)
   getTok(p)
   optInd(p, result)
   case p.tok.tokType
   of tkEof, tkSad, tkDed: addSon(result, ast.emptyNode)
   else: addSon(result, parseSymbol(p))
-  
+
-proc parseAs(p: var TParser): PNode =
-  result = newNodeP(nkPatternStmt, p)
-  getTok(p)                 # skip `as`
-  if p.tok.tokType == tkColon:
-    eat(p, tkColon)
-    addSon(result, parseStmt(p))
-  else:
-    addSon(result, parseExpr(p))
-  
 proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
   result = newNodeP(kind, p)
   while true:
@@ -1175,17 +1170,24 @@ proc parseGenericParamList(p: var TParser): PNode =
   optPar(p)
   eat(p, tkBracketRi)
 
+proc parsePattern(p: var TParser): PNode =
+  eat(p, tkCurlyLe)
+  result = parseStmt(p)
+  eat(p, tkCurlyRi)
+
 proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = 
   result = newNodeP(kind, p)
   getTok(p)
   optInd(p, result)
   addSon(result, identVis(p))
+  if p.tok.tokType == tkCurlyLe: addSon(result, parsePattern(p))
+  else: addSon(result, ast.emptyNode)
   if p.tok.tokType == tkBracketLe: addSon(result, parseGenericParamList(p))
   else: addSon(result, ast.emptyNode)
   addSon(result, parseParamList(p))
   if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p))
   else: addSon(result, ast.emptyNode)
-  # empty pattern:
+  # empty exception tracking:
   addSon(result, ast.emptyNode)
   if p.tok.tokType == tkEquals: 
     getTok(p)
@@ -1522,7 +1524,6 @@ proc complexOrSimpleStmt(p: var TParser): PNode =
   of tkWhen: result = parseIfOrWhen(p, nkWhenStmt)
   of tkVar: result = parseSection(p, nkVarSection, parseVariable)
   of tkBind: result = parseBind(p)
-  of tkAs: result = parseAs(p)
   else: result = simpleStmt(p)
   
 proc parseStmt(p: var TParser): PNode = 

compiler/patterns.nim

@@ -92,7 +92,15 @@ proc matches(c: PPatternContext, p, n: PNode): bool =
     of nkCharLit..nkInt64Lit: result = p.intVal == n.intVal
     of nkFloatLit..nkFloat64Lit: result = p.floatVal == n.floatVal
     of nkStrLit..nkTripleStrLit: result = p.strVal == n.strVal
-    of nkEmpty, nkNilLit, nkType: result = true
+    of nkEmpty, nkNilLit, nkType: 
+      result = true
+      # of nkStmtList:
+      # both are statement lists; we need to ignore comment statements and
+      # 'nil' statements and check whether p <: n which is however trivially
+      # checked as 'applyRule' is checked after every created statement
+      # already; We need to ensure that the matching span is passed to the
+      # macro and NOT simply 'n'!
+      # XXX
     else:
       if sonsLen(p) == sonsLen(n):
         for i in countup(0, sonsLen(p) - 1):
@@ -121,8 +129,6 @@ proc applyRule*(c: PContext, s: PSym, n: PNode): PNode =
       let param = params.sons[i].sym
       let x = IdNodeTableGetLazy(ctx.mapping, param)
       # couldn't bind parameter:
-      if isNil(x): 
+      if isNil(x): return nil
-        echo "couldn't bind ", param.name.s
-        return nil
       result.add(x)
-
+    markUsed(n, s)

compiler/pragmas.nim

@@ -41,7 +41,7 @@ const
     wFatal, wDefine, wUndef, wCompile, wLink, wLinkSys, wPure, wPush, wPop,
     wBreakpoint, wWatchpoint, wPassL, wPassC, wDeadCodeElim, wDeprecated,
     wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
-    wLinearScanEnd}
+    wLinearScanEnd, wPatterns}
   lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, 
     wNosideEffect, wSideEffect, wNoreturn, wDynLib, wHeader, 
     wDeprecated, wExtern, wThread, wImportcpp, wImportobjc, wNoStackFrame}
@@ -302,6 +302,7 @@ proc processOption(c: PContext, n: PNode) =
           excl(gOptions, optOptimizeSize)
         else: LocalError(n.info, errNoneSpeedOrSizeExpected)
     of wImplicitStatic: OnOff(c, n, {optImplicitStatic})
+    of wPatterns: OnOff(c, n, {optPatterns})
     else: LocalError(n.info, errOptionExpected)
   
 proc processPush(c: PContext, n: PNode, start: int) = 
@@ -643,7 +644,8 @@ proc pragma(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords) =
              wOverflowchecks, wNilchecks, wAssertions, wWarnings, wHints, 
              wLinedir, wStacktrace, wLinetrace, wOptimization,
              wCallConv, 
-             wDebugger, wProfiler, wFloatChecks, wNanChecks, wInfChecks: 
+             wDebugger, wProfiler, wFloatChecks, wNanChecks, wInfChecks,
+             wPatterns:
             processOption(c, it) # calling conventions (boring...):
           of firstCallConv..lastCallConv: 
             assert(sym != nil)

compiler/renderer.nim

@@ -588,13 +588,13 @@ proc gwhile(g: var TSrcGen, n: PNode) =
 
 proc gpattern(g: var TSrcGen, n: PNode) = 
   var c: TContext
-  put(g, tkAs, "as")
+  put(g, tkCurlyLe, "{")
-  putWithSpace(g, tkColon, ":")
   initContext(c)
-  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > maxLineLen): 
+  if longMode(n) or (lsub(n.sons[0]) + g.lineLen > maxLineLen):
     incl(c.flags, rfLongMode)
   gcoms(g)                    # a good place for comments
   gstmts(g, n.sons[0], c)
+  put(g, tkCurlyRi, "}")
 
 proc gpragmaBlock(g: var TSrcGen, n: PNode) = 
   var c: TContext
@@ -666,6 +666,7 @@ proc gproc(g: var TSrcGen, n: PNode) =
     put(g, tkSymbol, renderDefinitionName(n.sons[namePos].sym))
   else:
     gsub(g, n.sons[namePos])
+  gsub(g, n.sons[patternPos])
   gsub(g, n.sons[genericParamsPos])
   gsub(g, n.sons[paramsPos])
   gsub(g, n.sons[pragmasPos])
@@ -1045,7 +1046,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkCaseStmt, nkRecCase: gcase(g, n)
   of nkMacroStmt: gmacro(g, n)
   of nkTryStmt: gtry(g, n)
-  of nkPatternStmt: gpattern(g, n)
+  of nkPattern: gpattern(g, n)
   of nkForStmt, nkParForStmt: gfor(g, n)
   of nkBlockStmt, nkBlockExpr: gblock(g, n)
   of nkStaticStmt: gstaticStmt(g, n)

compiler/sem.nim

@@ -104,7 +104,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
 
 proc applyPatterns(c: PContext, n: PNode): PNode =
   # fast exit:
-  if c.patterns.len == 0: return n
+  if c.patterns.len == 0 or optPatterns notin gOptions: return n
   result = n
   # we apply the last pattern first, so that pattern overriding is possible;
   # however the resulting AST would better not trigger the old rule then
@@ -113,7 +113,11 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
     let x = applyRule(c, c.patterns[i], result)
     if not isNil(x):
       assert x.kind == nkCall
+      inc(evalTemplateCounter)
+      if evalTemplateCounter > 100:
+        GlobalError(n.info, errTemplateInstantiationTooNested)
       result = semExpr(c, x)
+      dec(evalTemplateCounter)
 
 include seminst, semcall
 

compiler/semstmts.nim

@@ -770,6 +770,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   else:
     s.typ = newTypeS(tyProc, c)
     rawAddSon(s.typ, nil)
+  if n.sons[patternPos].kind != nkEmpty:
+    n.sons[patternPos] = semPattern(c, n.sons[patternPos])
+    c.patterns.add(s)
+  
   var proto = SearchForProc(c, s, c.tab.tos-2) # -2 because we have a scope
                                                # open for parameters
   if proto == nil: 
@@ -1202,17 +1206,7 @@ proc SemStmt(c: PContext, n: PNode): PNode =
     result = semPragmaBlock(c, n)
   of nkStaticStmt:
     result = semStaticStmt(c, n)
-  of nkPatternStmt:
+  else:
-    let pat = semPatternStmt(c, n)
-    let s = getCurrOwner()
-    if s.kind in routineKinds and s.ast.sons[patternPos].kind == nkEmpty:
-      s.ast.sons[patternPos] = pat
-      c.patterns.add(s)
-    else:
-      LocalError(n.info, errXNotAllowedHere, "'as'")
-    # replace by an empty statement:
-    result = newNodeI(nkNilLit, n.info)
-  else: 
     # in interactive mode, we embed the expression in an 'echo':
     if gCmd == cmdInteractive:
       result = buildEchoStmt(c, semExpr(c, n))

compiler/semtempl.nim

@@ -172,10 +172,11 @@ proc semRoutineInTemplBody(c: var TemplCtx, n: PNode, k: TSymKind): PNode =
   else:
     n.sons[namePos] = semTemplBody(c, n.sons[namePos])
   openScope(c)
-  for i in genericParamsPos..bodyPos:
+  for i in patternPos..bodyPos:
     n.sons[i] = semTemplBody(c, n.sons[i])
   closeScope(c)
 
+proc semPattern(c: PContext, n: PNode): PNode
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode = 
   result = n
   case n.kind
@@ -396,7 +397,10 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
     s.typ.n = newNodeI(nkFormalParams, n.info)
     rawAddSon(s.typ, newTypeS(tyStmt, c))
     addSon(s.typ.n, newNodeIT(nkType, n.info, s.typ.sons[0]))
-      
+  if n.sons[patternPos].kind != nkEmpty:
+    n.sons[patternPos] = semPattern(c, n.sons[patternPos])
+    c.patterns.add(s)
+
   var ctx: TemplCtx
   ctx.toBind = initIntSet()
   ctx.c = c
@@ -422,7 +426,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   else:
     SymTabReplace(c.tab.stack[curScope], proto, s)
 
-proc semPatternStmt(c: PContext, n: PNode): PNode =
+proc semPattern(c: PContext, n: PNode): PNode =
   # not much to do here: We don't replace operators ``$``, ``*``, ``+``,
   # ``|``, ``~`` as meta operators and strip the leading ``\`` of all
   # operators.
@@ -432,5 +436,7 @@ proc semPatternStmt(c: PContext, n: PNode): PNode =
   ctx.c = c
   ctx.owner = getCurrOwner()
   ctx.bodyKind = bkPattern
-  result = semTemplBody(ctx, n.sons[0])
+  result = semTemplBody(ctx, n)
+  if result.kind in {nkStmtList, nkStmtListExpr} and result.len == 1:
+    result = result.sons[0]
   closeScope(c.tab)

compiler/wordrecg.nim

@@ -51,7 +51,7 @@ type
     wFastcall, wClosure, wNoconv, wOn, wOff, wChecks, wRangechecks, 
     wBoundchecks, wOverflowchecks, wNilchecks,
     wFloatchecks, wNanChecks, wInfChecks,
-    wAssertions, wWarnings, 
+    wAssertions, wPatterns, wWarnings,
     wHints, wOptimization, wSpeed, wSize, wNone, 
     wDeadCodeElim, wSafecode, 
     wPragma,
@@ -131,7 +131,7 @@ const
     "overflowchecks", "nilchecks",
     "floatchecks", "nanchecks", "infchecks",
 
-    "assertions", "warnings", "hints", 
+    "assertions", "patterns", "warnings", "hints", 
     "optimization", "speed", "size", "none", 
     "deadcodeelim", "safecode", 
     "pragma",

doc/advopt.txt

@@ -55,6 +55,7 @@ Advanced options:
   --taintMode:on|off        turn taint mode on|off
   --symbolFiles:on|off      turn symbol files on|off (experimental)
   --implicitStatic:on|off   turn implicit compile time evaluation on|off
+  --patterns:on|off         turn pattern matching on|off
   --skipCfg                 do not read the general configuration file
   --skipUserCfg             do not read the user's configuration file
   --skipParentCfg           do not read the parent dirs' configuration files

lib/core/macros.nim

@@ -46,7 +46,7 @@ type
     nnkYieldStmt, nnkTryStmt, nnkFinally, nnkRaiseStmt, 
     nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt,
     nnkDiscardStmt, nnkStmtList, nnkImportStmt, nnkFromStmt, 
-    nnkIncludeStmt, nnkBindStmt, nnkPatternStmt,
+    nnkIncludeStmt, nnkBindStmt, nnkPattern,
     nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr, 
     nnkStmtListType, nnkBlockType, nnkTypeOfExpr, nnkObjectTy, 
     nnkTupleTy, nnkRecList, nnkRecCase, nnkRecWhen, 

lib/pure/unittest.nim

@@ -92,7 +92,9 @@ template fail* =
   TestStatusIMPL = FAILED
   checkpoints = @[]
 
-macro check*(conditions: stmt): stmt =
+macro check*(conditions: stmt): stmt =
+  let conditions = callsite()
+  
   proc standardRewrite(e: PNimrodNode): PNimrodNode =
     template rewrite(Exp, lineInfoLit: expr, expLit: string): stmt =
       if not Exp:

tests/reject/tdisallowif.nim

@@ -0,0 +1,28 @@
+discard """
+  line: 24
+  errormsg: "usage of 'disallowIf' is a user-defined error"
+"""
+
+template optZero{x+x}(x: int): int = x*3
+template andthen{x*3}(x: int): int = x*4
+template optSubstr1{x = substr(x, a, b)}(x: string, a, b: int) = setlen(x, b+1)
+
+template disallowIf{
+  if cond: action
+  else: action2
+}(cond: bool, action, action2: stmt) {.error.} = action
+
+var y = 12
+echo y+y
+
+var s: array[0..2, string]
+s[0] = "hello"
+s[0] = substr(s[0], 0, 2)
+
+echo s[0]
+
+if true:
+  echo "do it"
+  echo "more branches"
+else:
+  nil

tests/run/tpatterns.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''48
+hel'''
+"""
+
+template optZero{x+x}(x: int): int = x*3
+template andthen{x*3}(x: int): int = x*4
+template optSubstr1{x = substr(x, a, b)}(x: string, a, b: int) = setlen(x, b+1)
+
+var y = 12
+echo y+y
+
+var s: array[0..2, string]
+s[0] = "hello"
+s[0] = substr(s[0], 0, 2)
+
+echo s[0]

todo.txt

@@ -3,7 +3,7 @@ version 0.9.0
 
 - make 'm: stmt' use overloading resolution
 
-- implement the high level optimizer
+- improve pattern matching: introduce meta operators, statement list support
 - make 'bind' default for templates and introduce 'mixin'
 
 - implement "closure tuple consists of a single 'ref'" optimization