getAst works correctly with existing AST values as template/macro arguments

compiler/evals.nim

@@ -65,6 +65,7 @@ proc popStackFrame*(c: PEvalContext) {.inline.} =
   if (c.tos == nil): InternalError("popStackFrame")
   c.tos = c.tos.next
 
+proc eval*(c: PEvalContext, n: PNode): PNode
 proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode
 
 proc stackTraceAux(x: PStackFrame) =
@@ -764,7 +765,7 @@ proc isEmpty(n: PNode): bool =
 proc stringStartingLine(s: PNode): int =
   result = s.info.line - countLines(s.strVal)
 
-proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
+proc evalParseExpr(c: PEvalContext, n: PNode): PNode =
   var code = evalAux(c, n.sons[1], {})
   var ast = parseString(code.getStrValue, code.info.toFilename,
                         code.stringStartingLine)
@@ -773,12 +774,59 @@ proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
   result = ast.sons[0]
   result.typ = newType(tyExpr, c.module)
 
-proc evalParseStmt(c: PEvalContext, n: Pnode): Pnode =
+proc evalParseStmt(c: PEvalContext, n: PNode): PNode =
   var code = evalAux(c, n.sons[1], {})
   result = parseString(code.getStrValue, code.info.toFilename,
                        code.stringStartingLine)
   result.typ = newType(tyStmt, c.module)
 
+proc evalMacroCall*(c: PEvalContext, n: PNode, sym: PSym): PNode =
+  var s = newStackFrame()
+  s.call = n
+  setlen(s.params, 2)
+  s.params[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
+  s.params[1] = n
+  pushStackFrame(c, s)
+  discard eval(c, sym.ast.sons[codePos])
+  result = s.params[0]
+  popStackFrame(c)
+  if cyclicTree(result): GlobalError(n.info, errCyclicTree)
+
+# XXX:
+# These imports could be removed when the template evaluation code is extracted in a 
+# separate module. semdata is needed only for PContext (which is not wanted here, see below)
+import
+  semdata, sem
+  
+proc evalExpandToAst(c: PEvalContext, n: PNode): PNode =
+  var 
+    macroCall = n.sons[1]
+    expandedSym = macroCall.sons[0].sym
+    
+    # XXX: It's unfortunate that evalTemplate requires a PContext,
+    # although it's used only for very specific corner cases.
+    #
+    # Template expansion should be about AST manipulation only, so 
+    # maybe this requirement can be lifted.
+    dummyContext : PContext
+
+  for i in countup(1, macroCall.sonsLen - 1):
+    macroCall.sons[i] = evalAux(c, macroCall.sons[i], {})
+
+  case expandedSym.kind
+  of skTemplate:
+    result = evalTemplate(dummyContext, macroCall, expandedSym)
+  of skMacro:
+    # XXX:
+    # At this point macroCall.sons[0] is nkSym node.
+    # To be completely compatible with normal macro invocation,
+    # we may want to replace it with nkIdent node featuring
+    # the original unmangled macro name.
+    result = evalMacroCall(c, macroCall, expandedSym)
+  else:
+    InternalError(macroCall.info,
+      "ExpandToAst: expanded symbol is no macro or template")
+
 proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode = 
   var m = getMagic(n)
   case m
@@ -805,6 +853,7 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
   of mAppendSeqElem: result = evalAppendSeqElem(c, n)
   of mParseExprToAst: result = evalParseExpr(c, n)
   of mParseStmtToAst: result = evalParseStmt(c, n)
+  of mExpandMacroToAst: result = evalExpandToAst(c, n)
   of mNLen: 
     result = evalAux(c, n.sons[1], {efLValue})
     if isSpecial(result): return 

compiler/sem.nim

@@ -10,29 +10,15 @@
 # This module implements the semantic checking pass.
 
 import
-  strutils, hashes, lists, options, lexer, ast, astalgo, trees, treetab, 
+  strutils, hashes, lists, options, lexer, ast, astalgo, trees, treetab,
-  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math, 
+  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
-  magicsys, parser, nversion, nimsets, semdata, evals, semfold, importer, 
+  magicsys, parser, nversion, semdata, nimsets, semfold, importer,
   procfind, lookups, rodread, pragmas, passes, semtypinst, sigmatch, suggest,
-  semthreads, intsets, transf
+  semthreads, intsets, transf, evals
 
 proc semPass*(): TPass
 # implementation
 
-proc isTopLevel(c: PContext): bool {.inline.} = 
-  result = c.tab.tos <= 2
-
-proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = 
-  result = newSym(kind, considerAcc(n), getCurrOwner())
-  result.info = n.info
-  
-proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
-                 allowed: TSymFlags): PSym
-  # identifier with visability
-proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, 
-                        allowed: TSymFlags): PSym
-proc semStmtScope(c: PContext, n: PNode): PNode
-
 type 
   TExprFlag = enum 
     efAllowType, efLValue, efWantIterator, efInTypeof
@@ -50,10 +36,36 @@ proc addResult(c: PContext, t: PType, info: TLineInfo)
 proc addResultNode(c: PContext, n: PNode)
 proc instGenericContainer(c: PContext, n: PNode, header: PType): PType
 
+proc typeMismatch(n: PNode, formal, actual: PType) = 
+  GlobalError(n.Info, errGenerated, msgKindToString(errTypeMismatch) &
+      typeToString(actual) & ") " &
+      `%`(msgKindToString(errButExpectedX), [typeToString(formal)]))
+
+proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = 
+  result = IndexTypesMatch(c, formal, arg.typ, arg)
+  if result == nil:
+    typeMismatch(arg, formal, arg.typ)
+
+proc isTopLevel(c: PContext): bool {.inline.} = 
+  result = c.tab.tos <= 2
+
+proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = 
+  result = newSym(kind, considerAcc(n), getCurrOwner())
+  result.info = n.info
+  
+proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
+                 allowed: TSymFlags): PSym
+  # identifier with visability
+proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, 
+                        allowed: TSymFlags): PSym
+proc semStmtScope(c: PContext, n: PNode): PNode
+
 proc ParamsTypeCheck(c: PContext, typ: PType) {.inline.} =
   if not typeAllowed(typ, skConst):
     GlobalError(typ.n.info, errXisNoType, typeToString(typ))
 
+include semtempl
+
 proc semConstExpr(c: PContext, n: PNode): PNode = 
   var e = semExprWithType(c, n)
   if e == nil: 
@@ -76,17 +88,7 @@ proc semAndEvalConstExpr(c: PContext, n: PNode): PNode =
   result = semConstExpr(c, n)
   
 include seminst, semcall
-  
+    
-proc typeMismatch(n: PNode, formal, actual: PType) = 
-  GlobalError(n.Info, errGenerated, msgKindToString(errTypeMismatch) &
-      typeToString(actual) & ") " &
-      `%`(msgKindToString(errButExpectedX), [typeToString(formal)]))
-
-proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = 
-  result = IndexTypesMatch(c, formal, arg.typ, arg)
-  if result == nil:
-    typeMismatch(arg, formal, arg.typ)
-  
 proc semAfterMacroCall(c: PContext, n: PNode, s: PSym): PNode = 
   result = n
   case s.typ.sons[0].kind
@@ -101,8 +103,6 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym): PNode =
     result = semExpr(c, result)
     result = fitNode(c, s.typ.sons[0], result)
     #GlobalError(s.info, errInvalidParamKindX, typeToString(s.typ.sons[0]))
-  
-include "semtempl.nim"
 
 proc semMacroExpr(c: PContext, n: PNode, sym: PSym, 
                   semCheck: bool = true): PNode = 
@@ -111,16 +111,7 @@ proc semMacroExpr(c: PContext, n: PNode, sym: PSym,
     GlobalError(n.info, errTemplateInstantiationTooNested)
   markUsed(n, sym)
   var p = newEvalContext(c.module, "", false)
-  var s = newStackFrame()
+  result = evalMacroCall(p, n, sym)
-  s.call = n
-  setlen(s.params, 2)
-  s.params[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
-  s.params[1] = n
-  pushStackFrame(p, s)
-  discard eval(p, sym.ast.sons[codePos])
-  result = s.params[0]
-  popStackFrame(p)
-  if cyclicTree(result): GlobalError(n.info, errCyclicTree)
   if semCheck: result = semAfterMacroCall(c, result, sym)
   dec(evalTemplateCounter)
 

compiler/semexprs.nim

@@ -904,26 +904,52 @@ proc expectStringArg(c: PContext, n: PNode, i: int): PNode =
   if result.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}:
     GlobalError(result.info, errStringLiteralExpected)
 
-proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+proc isAstValue(n: PNode): bool =
+  result = n.typ.sym.name.s in [ "expr", "stmt", "PNimrodNode" ]
+
+proc semExpandMacroToAst(c: PContext, n: PNode, magicSym: PSym, flags: TExprFlags): PNode =
   if sonsLen(n) == 2:
     if not isCallExpr(n.sons[1]):
       GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
 
     var macroCall = n.sons[1]
 
-    var s = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
+    var expandedSym = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
-    if s == nil:
+    if expandedSym == nil:
       GlobalError(n.info, errUndeclaredIdentifier, macroCall.sons[0].renderTree)
 
-    var expanded : Pnode
+    if not (expandedSym.kind in { skMacro, skTemplate }):
+      GlobalError(n.info, errXisNoMacroOrTemplate, expandedSym.name.s)
 
-    case s.kind
+    macroCall.sons[0] = newNodeI(nkSym, macroCall.info)
-    of skMacro: expanded = semMacroExpr(c, macroCall, s, false)
+    macroCall.sons[0].sym = expandedSym
-    of skTemplate: expanded = semTemplateExpr(c, macroCall, s, false)
+    markUsed(n, expandedSym)
-    else: GlobalError(n.info, errXisNoMacroOrTemplate, s.name.s)
 
-    var macroRetType = newTypeS(s.typ.sons[0].kind, c)
+    # Any macro arguments that are already AST values are passed as such
-    result = newMetaNodeIT(expanded, n.info, macroRetType)
+    # All other expressions within the arguments are converted to AST as
+    # in normal macro/template expansion.
+    # The actual expansion does not happen here, but in evals.nim, where
+    # the dynamic AST values will be known.
+    for i in countup(1, macroCall.sonsLen - 1):
+      var argAst = macroCall.sons[i]
+      var typedArg = semExprWithType(c, argAst, {efAllowType})
+      if isAstValue(typedArg):
+        macroCall.sons[i] = typedArg
+      else:
+        macroCall.sons[i] = newMetaNodeIT(argAst, argAst.info, newTypeS(tyExpr, c))
+
+    # Preserve the magic symbol in order to handled in evals.nim
+    n.sons[0] = newNodeI(nkSym, n.info)
+    n.sons[0].sym = magicSym
+    
+    # XXX: 
+    # Hmm, expandedSym.typ is something like proc (e: expr): stmt
+    # In theory, it should be better here to report the actual return type,
+    # but the code is working fine so far with tyStmt, so I am leaving it
+    # here for someone more knowledgable to see ;)
+    n.typ = newTypeS(tyStmt, c) # expandedSym.typ
+
+    result = n
   else:
     result = semDirectOp(c, n, flags)
 
@@ -963,7 +989,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     else:
       result = semDirectOp(c, n, flags)
   of mSlurp: result = semSlurp(c, n, flags)
-  of mExpandMacroToAst: result = semExpandMacroToAst(c, n, flags)
+  of mExpandMacroToAst: result = semExpandMacroToAst(c, n, s, flags)
   else: result = semDirectOp(c, n, flags)
 
 proc semIfExpr(c: PContext, n: PNode): PNode = 

compiler/semfold.nim

@@ -206,7 +206,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mNewString, mNewStringOfCap, 
      mExit, mInc, ast.mDec, mEcho, mAssert, mSwap, mAppendStrCh, 
      mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq, 
-     mParseExprToAst, mParseStmtToAst,
+     mParseExprToAst, mParseStmtToAst, mExpandMacroToAst,
      mNLen..mNError, mEqRef: 
     nil
   else: InternalError(a.info, "evalOp(" & $m & ')')

compiler/semtempl.nim

@@ -33,7 +33,7 @@ proc isTypeDesc(n: PNode): bool =
     result = true
   else: result = false
   
-proc evalTemplateAux(c: PContext, templ, actual: PNode, sym: PSym): PNode = 
+proc evalTemplateAux(templ, actual: PNode, sym: PSym): PNode = 
   case templ.kind
   of nkSym: 
     var p = templ.sym
@@ -47,7 +47,7 @@ proc evalTemplateAux(c: PContext, templ, actual: PNode, sym: PSym): PNode =
     result = copyNode(templ)
     newSons(result, sonsLen(templ))
     for i in countup(0, sonsLen(templ) - 1): 
-      result.sons[i] = evalTemplateAux(c, templ.sons[i], actual, sym)
+      result.sons[i] = evalTemplateAux(templ.sons[i], actual, sym)
   
 var evalTemplateCounter: int = 0
   # to prevend endless recursion in templates instantation
@@ -77,13 +77,13 @@ proc evalTemplateArgs(c: PContext, n: PNode, s: PSym): PNode =
       arg = fitNode(c, s.typ.sons[i], semExprWithType(c, arg))
     addSon(result, arg)
 
-proc evalTemplate(c: PContext, n: PNode, sym: PSym): PNode = 
+proc evalTemplate*(c: PContext, n: PNode, sym: PSym): PNode = 
   var args: PNode
   inc(evalTemplateCounter)
   if evalTemplateCounter <= 100: 
     # replace each param by the corresponding node:
     args = evalTemplateArgs(c, n, sym)
-    result = evalTemplateAux(c, sym.ast.sons[codePos], args, sym)
+    result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
     dec(evalTemplateCounter)
   else:
     GlobalError(n.info, errTemplateInstantiationTooNested)