Initial implementation of the parseExpr, parseStmt, getAst macro routines:

proc parseExpr*(s: string) : expr {.magic: "ParseExprToAst".}
  ## Compiles the passed string to its AST representation
  ## Expects a single expression

proc parseStmt*(s: string) : stmt {.magic: "ParseStmtToAst".}
  ## Compiles the passed string to its AST representation
  ## Expects one or more statements

proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".}
  ## Obtains the AST nodes returned from a macro or template invocation
  ## example:
  ## macro FooMacro() = 
  ##   var ast = getAst(BarTemplate())

Handling of the node.toYaml magic moved to the evaluation engine.

compiler/ast.nim

@@ -57,10 +57,10 @@ type
     nkStrLit,             # a string literal ""
     nkRStrLit,            # a raw string literal r""
     nkTripleStrLit,       # a triple string literal """
-    nkMetaNode,           # difficult to explain; represents itself
-                          # (used for macros)
     nkNilLit,             # the nil literal
                           # end of atoms
+    nkMetaNode,           # difficult to explain; represents itself
+                          # (used for macros)
     nkDotCall,            # used to temporarily flag a nkCall node; 
                           # this is used
                           # for transforming ``s.len`` to ``len(s)``
@@ -312,7 +312,8 @@ type
 
   TMagic* = enum # symbols that require compiler magic:
     mNone, mDefined, mDefinedInScope, mLow, mHigh, mSizeOf, mIs, mOf,
-    mEcho, mAstToYaml, mShallowCopy, mSlurp,
+    mEcho, mShallowCopy, mSlurp,
+    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandMacroToAst,
     mUnaryLt, mSucc, 
     mPred, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray, 
     mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr, mGCref, 
@@ -610,6 +611,23 @@ proc copyTree*(src: PNode): PNode
 
 proc discardSons*(father: PNode)
 
+proc len*(n: PNode): int {.inline.} =
+  if isNil(n.sons): result = 0
+  else: result = len(n.sons)
+  
+proc safeLen*(n: PNode): int {.inline.} =
+  ## works even for leaves.
+  if n.kind in {nkNone..nkNilLit} or isNil(n.sons): result = 0
+  else: result = len(n.sons)
+  
+proc add*(father, son: PNode) =
+  assert son != nil
+  if isNil(father.sons): father.sons = @[]
+  add(father.sons, son)
+  
+proc `[]`*(n: PNode, i: int): PNode {.inline.} =
+  result = n.sons[i]
+
 var emptyNode* = newNode(nkEmpty)
 # There is a single empty node that is shared! Do not overwrite it!
 
@@ -754,6 +772,10 @@ proc newNodeIT(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
   result.info = info
   result.typ = typ
 
+proc newMetaNodeIT*(tree: PNode, info: TLineInfo, typ: PType): PNode =
+  result = newNodeIT(nkMetaNode, info, typ)
+  result.add(tree)
+
 proc NewType(kind: TTypeKind, owner: PSym): PType = 
   new(result)
   result.kind = kind
@@ -866,23 +888,6 @@ proc sonsLen(n: PNode): int =
   if isNil(n.sons): result = 0
   else: result = len(n.sons)
   
-proc len*(n: PNode): int {.inline.} =
-  if isNil(n.sons): result = 0
-  else: result = len(n.sons)
-  
-proc safeLen*(n: PNode): int {.inline.} =
-  ## works even for leaves.
-  if n.kind in {nkNone..nkNilLit} or isNil(n.sons): result = 0
-  else: result = len(n.sons)
-  
-proc add*(father, son: PNode) =
-  assert son != nil
-  if isNil(father.sons): father.sons = @[]
-  add(father.sons, son)  
-  
-proc `[]`*(n: PNode, i: int): PNode {.inline.} =
-  result = n.sons[i]
-  
 proc newSons(father: PNode, length: int) = 
   if isNil(father.sons): father.sons = @[]
   setlen(father.sons, len(father.sons) + length)

compiler/evals.nim

@@ -15,7 +15,8 @@
 
 import 
   strutils, magicsys, lists, options, ast, astalgo, trees, treetab, nimsets, 
-  msgs, os, condsyms, idents, renderer, types, passes, semfold, transf
+  msgs, os, condsyms, idents, renderer, types, passes, semfold, transf, 
+  ropes
 
 type 
   PStackFrame* = ref TStackFrame
@@ -981,6 +982,9 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
     if (a == b) or
         (b.kind in {nkNilLit, nkEmpty}) and (a.kind in {nkNilLit, nkEmpty}): 
       result.intVal = 1
+  of mAstToYaml:
+    var ast = evalAux(c, n.sons[1], {efLValue})
+    result = newStrNode(nkStrLit, ast.treeToYaml.ropeToStr)
   of mNHint: 
     result = evalAux(c, n.sons[1], {})
     if isSpecial(result): return 
@@ -1034,6 +1038,9 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
   dec(gNestedEvals)
   if gNestedEvals <= 0: stackTrace(c, n, errTooManyIterations)
   case n.kind                 # atoms:
+  of nkMetaNode:
+    result = copyTree(n.sons[0])
+    result.typ = n.typ
   of nkEmpty: result = n
   of nkSym: result = evalSym(c, n, flags)
   of nkType..nkNilLit: result = copyNode(n) # end of atoms

compiler/parser.nim

@@ -30,6 +30,11 @@ proc closeParser*(p: var TParser)
 proc parseTopLevelStmt*(p: var TParser): PNode
   # implements an iterator. Returns the next top-level statement or
   # emtyNode if end of stream.
+
+proc parseString*(s: string, filename: string = "", line: int = 0): PNode
+  # filename and line could be set optionally, when the string originates 
+  # from a certain source file. This way, the compiler could generate
+  # correct error messages referring to the original source.
   
 # helpers for the other parsers
 proc getPrecedence*(tok: TToken): int
@@ -1369,3 +1374,13 @@ proc parseTopLevelStmt(p: var TParser): PNode =
       result = complexOrSimpleStmt(p)
       if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok)
       break
+
+proc parseString(s: string, filename: string = "", line: int = 0): PNode =
+  var stream = LLStreamOpen(s)
+  stream.lineOffset = line
+
+  var parser : TParser
+  OpenParser(parser, filename, stream)
+  
+  result = parser.parseAll
+  

compiler/semexprs.nim

@@ -385,6 +385,10 @@ proc isAssignable(c: PContext, n: PNode): TAssignableResult =
   else: 
     nil
 
+proc isCallExpr(n: PNode): bool = 
+  result = n.kind in {nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand,
+                      nkCallStrLit}
+
 proc newHiddenAddrTaken(c: PContext, n: PNode): PNode = 
   if n.kind == nkHiddenDeref: 
     checkSonsLen(n, 1)
@@ -884,16 +888,66 @@ proc setMs(n: PNode, s: PSym): PNode =
   n.sons[0] = newSymNode(s)
   n.sons[0].info = n.info
 
-proc semAstToYaml(c: PContext, n: PNode): PNode =
+proc expectStringArg(c: PContext, n: PNode, i: int): PNode =
-  result = newStrNode(nkStrLit, n.treeToYaml.ropeToStr)
+  result = c.semConstExpr(c, n.sons[i+1])
-  result.typ = getSysType(tyString)
+
-  result.info = n.info
+  if result.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}:
+     GlobalError(result.info, errStringLiteralExpected)
+
+# The lexer marks multi-line strings as residing at the line where they are closed
+# This function returns the line where the string begins
+# Maybe the lexer should mark both the beginning and the end of expressions, then
+# this function could be removed
+proc stringStartingLine(s: PNode): int =
+  var totalLines = 0
+  for ln in splitLines(s.strVal): inc totalLines
+
+  result = s.info.line - totalLines
+
+proc semParseExprToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+  if sonsLen(n) == 2:
+    var code = expectStringArg(c, n, 0)
+    var ast = parseString(code.strVal, code.info.toFilename, code.stringStartingLine)
+
+    if sonsLen(ast) != 1:
+      GlobalError(code.info, errExprExpected, "multiple statements")
+
+    result = newMetaNodeIT(ast.sons[0], code.info, newTypeS(tyExpr, c))
+  else:
+    result = semDirectOp(c, n, flags)
+
+proc semParseStmtToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+  if sonsLen(n) == 2:
+    var code = expectStringArg(c, n, 0)
+    var ast = parseString(code.strVal, code.info.toFilename, code.stringStartingLine)
+
+    result = newMetaNodeIT(ast, code.info, newTypeS(tyStmt, c))
+  else:
+    result = semDirectOp(c, n, flags)
+
+proc semExpandMacroToAst(c: PContext, n: PNode, flags: TExprFlags): PNode =
+  if sonsLen(n) == 2 and isCallExpr(n.sons[1]):
+    var macroCall = n.sons[1]
+
+    var s = qualifiedLookup(c, macroCall.sons[0], {checkUndeclared})
+    if s == nil:
+      GlobalError(n.info, errUndeclaredIdentifier, macroCall.sons[0].strVal)
+
+    var expanded : Pnode
+
+    case s.kind
+    of skMacro: expanded = semMacroExpr(c, macroCall, s, false)
+    of skTemplate: expanded = semTemplateExpr(c, macroCall, s, false)
+    else: GlobalError(n.info, errXisNoMacroOrTemplate, s.name.s)
+
+    var macroRetType = newTypeS(s.typ.sons[0].kind, c)
+    result = newMetaNodeIT(expanded, n.info, macroRetType)
+  else:
+    GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
 
 proc semSlurp(c: PContext, n: PNode, flags: TExprFlags): PNode = 
   if sonsLen(n) == 2:
-    var a = c.semConstExpr(c, n.sons[1])
+    var a = expectStringArg(c, n, 0)
-    if a.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}: 
-      GlobalError(a.info, errStringLiteralExpected)
     try:
       var content = readFile(a.strVal)
       result = newStrNode(nkStrLit, content)
@@ -927,7 +981,9 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     else:
       result = semDirectOp(c, n, flags)
   of mSlurp: result = semSlurp(c, n, flags)
-  of mAstToYaml: result = semAstToYaml(c, n)
+  of mParseExprToAst: result = semParseExprToAst(c, n, flags)
+  of mParseStmtToAst: result = semParseStmtToAst(c, n, flags)
+  of mExpandMacroToAst: result = semExpandMacroToAst(c, n, flags)
   else: result = semDirectOp(c, n, flags)
 
 proc semIfExpr(c: PContext, n: PNode): PNode = 
@@ -1085,10 +1141,6 @@ proc semBlockExpr(c: PContext, n: PNode): PNode =
   closeScope(c.tab)
   Dec(c.p.nestedBlockCounter)
 
-proc isCallExpr(n: PNode): bool = 
-  result = n.kind in {nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand,
-                      nkCallStrLit}
-
 proc semMacroStmt(c: PContext, n: PNode, semCheck = true): PNode = 
   checkMinSonsLen(n, 2)
   var a: PNode

lib/core/macros.nim

@@ -19,7 +19,7 @@ type
     nnkType, nnkCharLit, nnkIntLit, nnkInt8Lit, 
     nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkFloatLit, 
     nnkFloat32Lit, nnkFloat64Lit, nnkStrLit, nnkRStrLit, 
-    nnkTripleStrLit, nnkMetaNode, nnkNilLit, nnkDotCall, 
+    nnkTripleStrLit, nnkNilLit, nnkMetaNode, nnkDotCall, 
     nnkCommand, nnkCall, nnkCallStrLit, nnkExprEqExpr, 
     nnkExprColonExpr, nnkIdentDefs, nnkVarTuple, nnkInfix, 
     nnkPrefix, nnkPostfix, nnkPar, nnkCurly, 
@@ -211,12 +211,26 @@ proc prettyPrint*(n: PNimrodNode): string {.compileTime.} =
 
   add(result, ")")
 
-proc toYaml*(n: PNimrodNode) {.magic: "AstToYaml".}
+proc toYaml*(n: PNimrodNode): string {.magic: "AstToYaml".}
   ## Converts the AST `n` to an YAML string
   ##
   ## Provides more detailed, potentially harder to digest information
   ## than `prettyPrint`
 
+proc parseExpr*(s: string) : expr {.magic: "ParseExprToAst".}
+  ## Compiles the passed string to its AST representation
+  ## Expects a single expression
+
+proc parseStmt*(s: string) : stmt {.magic: "ParseStmtToAst".}
+  ## Compiles the passed string to its AST representation
+  ## Expects one or more statements
+
+proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".}
+  ## Obtains the AST nodes returned from a macro or template invocation
+  ## example:
+  ## macro FooMacro() = 
+  ##   var ast = getAst(BarTemplate())
+  
 proc expectKind*(n: PNimrodNode, k: TNimrodNodeKind) {.compileTime.} =
   ## checks that `n` is of kind `k`. If this is not the case,
   ## compilation aborts with an error message. This is useful for writing

tests/accept/compile/tdumpast.nim

@@ -1,12 +1,27 @@
 # Dump the contents of a PNimrodNode
 
 import macros
+
+template plus(a, b: expr): expr =
+  a + b
+
+macro call(e: expr): expr =
+  return newCall("foo", newStrLitNode("bar"))
   
-macro dumpAST(n: stmt): stmt = 
+macro dumpAST(n: stmt): stmt =
   # dump AST as a side-effect and return the inner node
   echo n.prettyPrint
   echo n.toYaml
 
+  var plusAst = getAst(plus(1, 2))
+  echo plusAst.prettyPrint
+
+  var callAst = getAst(call())
+  echo callAst.prettyPrint
+
+  var e = parseExpr("foo(bar + baz)")
+  echo e.prettyPrint
+
   result = n[1]
   
 dumpAST:
@@ -15,4 +30,3 @@ dumpAST:
   
   proc sub(x, y: int): int = return x - y
 
-