Cleaned up the circular dependecies and remaining issues

Changed: The []= operator for strings and sequences is now capable of splicing

compiler/ast.nim

@@ -327,7 +327,7 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone, mDefined, mDefinedInScope, mLow, mHigh, mSizeOf, mIs, mOf,
     mEcho, mShallowCopy, mSlurp,
-    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandMacroToAst,
+    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandToAst,
     mUnaryLt, mSucc, 
     mPred, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray, 
     mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr, mGCref, 

compiler/evals.nim

@@ -780,7 +780,66 @@ proc evalParseStmt(c: PEvalContext, n: PNode): PNode =
                        code.stringStartingLine)
   result.typ = newType(tyStmt, c.module)
 
+proc evalTemplateAux*(templ, actual: PNode, sym: PSym): PNode = 
+  case templ.kind
+  of nkSym: 
+    var p = templ.sym
+    if (p.kind == skParam) and (p.owner.id == sym.id): 
+      result = copyTree(actual.sons[p.position])
+    else: 
+      result = copyNode(templ)
+  of nkNone..nkIdent, nkType..nkNilLit: # atom
+    result = copyNode(templ)
+  else: 
+    result = copyNode(templ)
+    newSons(result, sonsLen(templ))
+    for i in countup(0, sonsLen(templ) - 1): 
+      result.sons[i] = evalTemplateAux(templ.sons[i], actual, sym)
+
+proc evalTemplateArgs(n: PNode, s: PSym): PNode =
+  var 
+    f, a: int
+    arg: PNode
+    
+  f = sonsLen(s.typ)
+
+  # if the template has zero arguments, it can be called without ``()``
+  # `n` is then a nkSym or something similar
+  case n.kind
+  of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
+    a = sonsLen(n)
+  else: a = 0
+  
+  if a > f: GlobalError(n.info, errWrongNumberOfArguments)
+
+  result = copyNode(n)
+  for i in countup(1, f - 1):
+    if i < a:
+      arg = n.sons[i]
+    else:
+      arg = copyTree(s.typ.n.sons[i].sym.ast)
+
+    addSon(result, arg)
+
+var evalTemplateCounter = 0
+  # to prevend endless recursion in templates instantation
+
+proc evalTemplate(n: PNode, sym: PSym): PNode = 
+  inc(evalTemplateCounter)
+  if evalTemplateCounter > 100:
+    GlobalError(n.info, errTemplateInstantiationTooNested)
+
+  # replace each param by the corresponding node:
+  var args = evalTemplateArgs(n, sym)
+  result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
+
+  dec(evalTemplateCounter)
+
 proc evalMacroCall*(c: PEvalContext, n: PNode, sym: PSym): PNode =
+  inc(evalTemplateCounter)
+  if evalTemplateCounter > 100: 
+    GlobalError(n.info, errTemplateInstantiationTooNested)
+
   var s = newStackFrame()
   s.call = n
   setlen(s.params, 2)
@@ -792,11 +851,7 @@ proc evalMacroCall*(c: PEvalContext, n: PNode, sym: PSym): PNode =
   popStackFrame(c)
   if cyclicTree(result): GlobalError(n.info, errCyclicTree)
 
-# XXX:
+  dec(evalTemplateCounter)
-# These imports could be removed when the template evaluation code is extracted in a 
-# separate module. semdata is needed only for PContext (which is not wanted here, see below)
-import
-  semdata, sem
   
 proc evalExpandToAst(c: PEvalContext, original: PNode): PNode =
   var
@@ -804,25 +859,18 @@ proc evalExpandToAst(c: PEvalContext, original: PNode): PNode =
     macroCall = n.sons[1]
     expandedSym = macroCall.sons[0].sym
 
-    # XXX: It's unfortunate that evalTemplate requires a PContext,
-    # although it's used only for very specific corner cases.
-    #
-    # Template expansion should be about AST manipulation only, so 
-    # maybe this requirement can be lifted.
-    dummyContext : PContext
-
   for i in countup(1, macroCall.sonsLen - 1):
     macroCall.sons[i] = evalAux(c, macroCall.sons[i], {})
 
   case expandedSym.kind
   of skTemplate:
-    result = evalTemplate(dummyContext, macroCall, expandedSym)
+    result = evalTemplate(macroCall, expandedSym)
   of skMacro:
-    # XXX:
     # At this point macroCall.sons[0] is nkSym node.
     # To be completely compatible with normal macro invocation,
-    # we may want to replace it with nkIdent node featuring
+    # we want to replace it with nkIdent node featuring
     # the original unmangled macro name.
+    macroCall.sons[0] = newIdentNode(expandedSym.name, expandedSym.info)
     result = evalMacroCall(c, macroCall, expandedSym)
   else:
     InternalError(macroCall.info,
@@ -854,7 +902,7 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
   of mAppendSeqElem: result = evalAppendSeqElem(c, n)
   of mParseExprToAst: result = evalParseExpr(c, n)
   of mParseStmtToAst: result = evalParseStmt(c, n)
-  of mExpandMacroToAst: result = evalExpandToAst(c, n)
+  of mExpandToAst: result = evalExpandToAst(c, n)
   of mNLen:
     result = evalAux(c, n.sons[1], {efLValue})
     if isSpecial(result): return 

compiler/sem.nim

@@ -106,14 +106,10 @@ proc semAfterMacroCall(c: PContext, n: PNode, s: PSym): PNode =
 
 proc semMacroExpr(c: PContext, n: PNode, sym: PSym, 
                   semCheck: bool = true): PNode = 
-  inc(evalTemplateCounter)
-  if evalTemplateCounter > 100: 
-    GlobalError(n.info, errTemplateInstantiationTooNested)
   markUsed(n, sym)
   var p = newEvalContext(c.module, "", false)
   result = evalMacroCall(p, n, sym)
   if semCheck: result = semAfterMacroCall(c, result, sym)
-  dec(evalTemplateCounter)
 
 proc forceBool(c: PContext, n: PNode): PNode = 
   result = fitNode(c, getSysType(tyBool), n)

compiler/semexprs.nim

@@ -907,7 +907,7 @@ proc expectStringArg(c: PContext, n: PNode, i: int): PNode =
 proc isAstValue(n: PNode): bool =
   result = n.typ.sym.name.s in [ "expr", "stmt", "PNimrodNode" ]
 
-proc semExpandMacroToAst(c: PContext, n: PNode, magicSym: PSym, flags: TExprFlags): PNode =
+proc semExpandToAst(c: PContext, n: PNode, magicSym: PSym, flags: TExprFlags): PNode =
   if sonsLen(n) == 2:
     if not isCallExpr(n.sons[1]):
       GlobalError(n.info, errXisNoMacroOrTemplate, n.renderTree)
@@ -925,29 +925,14 @@ proc semExpandMacroToAst(c: PContext, n: PNode, magicSym: PSym, flags: TExprFlag
     macroCall.sons[0].sym = expandedSym
     markUsed(n, expandedSym)
 
-    # Any macro arguments that are already AST values are passed as such
-    # All other expressions within the arguments are converted to AST as
-    # in normal macro/template expansion.
-    # The actual expansion does not happen here, but in evals.nim, where
-    # the dynamic AST values will be known.
     for i in countup(1, macroCall.sonsLen - 1):
-      var argAst = macroCall.sons[i]
+      macroCall.sons[i] = semExprWithType(c, macroCall.sons[i], {efAllowType})
-      var typedArg = semExprWithType(c, argAst, {efAllowType})
-      if isAstValue(typedArg):
-        macroCall.sons[i] = typedArg
-      else:
-        macroCall.sons[i] = newMetaNodeIT(argAst, argAst.info, newTypeS(tyExpr, c))
 
     # Preserve the magic symbol in order to handled in evals.nim
     n.sons[0] = newNodeI(nkSym, n.info)
     n.sons[0].sym = magicSym
    
-    # XXX: 
+    n.typ = expandedSym.getReturnType
-    # Hmm, expandedSym.typ is something like proc (e: expr): stmt
-    # In theory, it should be better here to report the actual return type,
-    # but the code is working fine so far with tyStmt, so I am leaving it
-    # here for someone more knowledgable to see ;)
-    n.typ = newTypeS(tyStmt, c) # expandedSym.typ
 
     result = n
   else:
@@ -989,7 +974,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     else:
       result = semDirectOp(c, n, flags)
   of mSlurp: result = semSlurp(c, n, flags)
-  of mExpandMacroToAst: result = semExpandMacroToAst(c, n, s, flags)
+  of mExpandToAst: result = semExpandToAst(c, n, s, flags)
   else: result = semDirectOp(c, n, flags)
 
 proc semIfExpr(c: PContext, n: PNode): PNode = 

compiler/semfold.nim

@@ -206,7 +206,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mNewString, mNewStringOfCap, 
      mExit, mInc, ast.mDec, mEcho, mAssert, mSwap, mAppendStrCh, 
      mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq, 
-     mParseExprToAst, mParseStmtToAst, mExpandMacroToAst,
+     mParseExprToAst, mParseStmtToAst, mExpandToAst,
      mNLen..mNError, mEqRef: 
     nil
   else: InternalError(a.info, "evalOp(" & $m & ')')

compiler/semstmts.nim

@@ -440,7 +440,7 @@ proc semTry(c: PContext, n: PNode): PNode =
     var length = sonsLen(a)
     if a.kind == nkExceptBranch:
       if length == 2 and a.sons[0].kind == nkBracket:
-        a.sons.splice(0, 1, a.sons[0].sons)
+        a.sons[0..0] = a.sons[0].sons
         length = a.sonsLen
 
       for j in countup(0, length - 2):

compiler/semtempl.nim

@@ -33,22 +33,6 @@ proc isTypeDesc(n: PNode): bool =
     result = true
   else: result = false
   
-proc evalTemplateAux(templ, actual: PNode, sym: PSym): PNode = 
-  case templ.kind
-  of nkSym: 
-    var p = templ.sym
-    if (p.kind == skParam) and (p.owner.id == sym.id): 
-      result = copyTree(actual.sons[p.position])
-    else: 
-      result = copyNode(templ)
-  of nkNone..nkIdent, nkType..nkNilLit: # atom
-    result = copyNode(templ)
-  else: 
-    result = copyNode(templ)
-    newSons(result, sonsLen(templ))
-    for i in countup(0, sonsLen(templ) - 1): 
-      result.sons[i] = evalTemplateAux(templ.sons[i], actual, sym)
-  
 var evalTemplateCounter: int = 0
   # to prevend endless recursion in templates instantation
 

compiler/types.nim

@@ -925,6 +925,11 @@ proc computeSize(typ: PType): biggestInt =
   var a: biggestInt = 1
   result = computeSizeAux(typ, a)
 
+proc getReturnType*(s: PSym): PType =
+  # Obtains the return type of a iterator/proc/macro/template
+  assert s.kind in { skProc, skTemplate, skMacro, skIterator }
+  result = s.typ.n.sons[0].typ
+
 proc getSize(typ: PType): biggestInt = 
   result = computeSize(typ)
   if result < 0: InternalError("getSize(" & $typ.kind & ')')

lib/core/macros.nim

@@ -230,7 +230,7 @@ proc parseStmt*(s: string): stmt {.magic: "ParseStmtToAst".}
   ## Compiles the passed string to its AST representation.
   ## Expects one or more statements.
 
-proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".}
+proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandToAst".}
   ## Obtains the AST nodes returned from a macro or template invocation.
   ## Example:
   ## 

lib/pure/unittest.nim

@@ -56,7 +56,7 @@ template test*(name: expr, body: stmt): stmt =
 
     finally:
       TestTeardownIMPL()
-      echo "[" & $TestStatusIMPL & "] " & name
+      echo "[" & $TestStatusIMPL & "] " & name & "\n"
 
 proc checkpoint*(msg: string) =
   checkpoints.add(msg)
@@ -78,18 +78,7 @@ macro check*(conditions: stmt): stmt =
         checkpoint(lineInfoLit & ": Check failed: " & expLit)
         fail()
  
-    # XXX: If we don't create a string literal node below, the compiler
+    result = getAst(rewrite(e, e.lineinfo, e.toStrLit))
-    # will SEGFAULT in a rather strange fashion:
-    #
-    # rewrite(e, e.toStrLit, e.toStrLit) is ok, but
-    #
-    # rewrite(e, e.lineinfo, e.toStrLit) or
-    # rewrite(e, "anything", e.toStrLit) are not
-    #
-    # It may have something to do with the dummyContext hack in 
-    # evals.nim/evalTemplate
-    #
-    result = getAst(rewrite(e, newStrLitNode(e.lineinfo), e.toStrLit))
   
   case conditions.kind
   of nnkCall, nnkCommand, nnkMacroStmt:
@@ -110,7 +99,7 @@ macro check*(conditions: stmt): stmt =
 
         result = getAst(rewrite(
           op[0], op[1], op[2],
-          newStrLitNode(op.lineinfo),
+          op.lineinfo,
           op.toStrLit,
           op[1].toStrLit,
           op[2].toStrLit))
@@ -153,5 +142,5 @@ macro expect*(exp: stmt): stmt =
   for i in countup(1, expectCall.len - 1):
     errorTypes.add(expectCall[i])
 
-  result = getAst(expectBody(errorTypes, newStrLitNode(exp.lineinfo), body))
+  result = getAst(expectBody(errorTypes, exp.lineinfo, body))
 

lib/system.nim

@@ -857,28 +857,6 @@ proc insert*[T](x: var seq[T], item: T, i = 0) {.noSideEffect.} =
     dec(j)
   x[i] = item
 
-template spliceImpl(x, start, count, elements: expr): stmt =
-  var 
-    shift = elements.len - count
-    newLen = x.len + shift
-    totalShifted = x.len - (start + count)
-    firstShifted = newLen - totalShifted
-    
-  if shift > 0:
-    setLen(x, newLen)
-
-  for i in countup(firstShifted, newLen - 1):
-    shallowCopy(x[i], x[i-shift])
-
-  for c in countup(0, elements.len - 1):
-    x[start + c] = elements[c]
-
-  if shift < 0:
-    setLen(x, newLen)
-
-proc splice*[T](x: var seq[T], start, count: int, elements: openarray[T] = []) =
-  spliceImpl(x, start, count, elements)
-
 proc repr*[T](x: T): string {.magic: "Repr", noSideEffect.}
   ## takes any Nimrod variable and returns its string representation. It
   ## works even for complex data graphs with cycles. This is a great
@@ -1943,6 +1921,26 @@ proc `[]`*(s: string, x: TSlice[int]): string {.inline.} =
   ## slice operation for strings. Negative indexes are supported.
   result = s.substr(x.a-|s, x.b-|s)
 
+template spliceImpl(x, start, endp, spliced: expr): stmt =
+  var 
+    count = endp - start + 1
+    shift = spliced.len - count
+    newLen = x.len + shift
+    totalShifted = x.len - (start + count)
+    firstShifted = newLen - totalShifted
+
+  if shift > 0:
+    setLen(x, newLen)
+
+  for i in countdown(newLen - 1, firstShifted):
+    shallowCopy(x[i], x[i-shift])
+
+  for c in countup(0, spliced.len - 1):
+    x[start + c] = spliced[c]
+
+  if shift < 0:
+    setLen(x, newLen)
+
 proc `[]=`*(s: var string, x: TSlice[int], b: string) = 
   ## slice assignment for strings. Negative indexes are supported.
   var a = x.a-|s
@@ -1950,7 +1948,7 @@ proc `[]=`*(s: var string, x: TSlice[int], b: string) =
   if L == b.len:
     for i in 0 .. <L: s[i+a] = b[i]
   else:
-    raise newException(EOutOfRange, "differing lengths for slice assignment")
+    spliceImpl(s, x.a, x.b, b)
 
 proc `[]`*[Idx, T](a: array[Idx, T], x: TSlice[int]): seq[T] =
   ## slice operation for arrays. Negative indexes are NOT supported because
@@ -2004,7 +2002,7 @@ proc `[]=`*[T](s: var seq[T], x: TSlice[int], b: openArray[T]) =
   if L == b.len:
     for i in 0 .. <L: s[i+a] = b[i]
   else:
-    raise newException(EOutOfRange, "differing lengths for slice assignment")
+    spliceImpl(s, x.a, x.b, b)
 
 proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo".}
   ## get type information for `x`. Ordinary code should not use this, but