distinguish properly between nkOpen and nkClosedSymChoice

compiler/ast.nim

@@ -104,7 +104,8 @@ type
 
     nkTableConstr,        # a table constructor {expr: expr}
     nkBind,               # ``bind expr`` node
-    nkSymChoice,          # symbol choice node
+    nkClosedSymChoice,    # symbol choice node; a list of nkSyms (closed)
+    nkOpenSymChoice,      # symbol choice node; a list of nkSyms (open)
     nkHiddenStdConv,      # an implicit standard type conversion
     nkHiddenSubConv,      # an implicit type conversion from a subtype
                           # to a supertype
@@ -473,6 +474,11 @@ const
     mInRange, mInSet, mRepr,
     mRand, 
     mCopyStr, mCopyStrLast}
+  # magics that require special semantic checking and
+  # thus cannot be overloaded (also documented in the spec!):
+  SpecialSemMagics* = {
+    mDefined, mDefinedInScope, mCompiles, mLow, mHigh, mSizeOf, mIs, mOf, 
+    mEcho, mShallowCopy, mExpandToAst}
 
 type 
   PNode* = ref TNode

compiler/lookups.nim

@@ -230,7 +230,7 @@ proc InitOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         LocalError(n.sons[1].info, errIdentifierExpected, 
                    renderTree(n.sons[1]))
         result = errorSym(c, n.sons[1])
-  of nkSymChoice: 
+  of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
     result = n.sons[0].sym
     o.stackPtr = 1
@@ -269,7 +269,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       result = n.sons[o.stackPtr].sym
       Incl(o.inSymChoice, result.id)
       inc(o.stackPtr)
-    else:
+    elif n.kind == nkOpenSymChoice:
       # try 'local' symbols too for Koenig's lookup:
       o.mode = oimSymChoiceLocalLookup
       o.stackPtr = c.tab.tos-1

compiler/renderer.nim

@@ -379,7 +379,8 @@ proc lsub(n: PNode): int =
   of nkPar, nkCurly, nkBracket, nkClosure: result = lcomma(n) + 2
   of nkTableConstr:
     result = if n.len > 0: lcomma(n) + 2 else: len("{:}")
-  of nkSymChoice: result = lsons(n) + len("()") + sonsLen(n) - 1
+  of nkClosedSymChoice, nkOpenSymChoice: 
+    result = lsons(n) + len("()") + sonsLen(n) - 1
   of nkTupleTy: result = lcomma(n) + len("tuple[]")
   of nkDotExpr: result = lsons(n) + 1
   of nkBind: result = lsons(n) + len("bind_")
@@ -833,7 +834,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkParLe, "(")
     gcomma(g, n, 1)
     put(g, tkParRi, ")")
-  of nkSymChoice: 
+  of nkClosedSymChoice, nkOpenSymChoice:
     put(g, tkParLe, "(")
     for i in countup(0, sonsLen(n) - 1): 
       if i > 0: put(g, tkOpr, "|")

compiler/semcall.nim

@@ -10,7 +10,7 @@
 ## This module implements semantic checking for calls. 
 # included from sem.nim
 
-proc sameMethodDispatcher(a, b: PSym): bool = 
+proc sameMethodDispatcher(a, b: PSym): bool =
   result = false
   if a.kind == skMethod and b.kind == skMethod: 
     var aa = lastSon(a.ast)
@@ -132,11 +132,11 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     if safeLen(s.ast.sons[genericParamsPos]) != n.len-1:
       return explicitGenericInstError(n)
     result = explicitGenericSym(c, n, s)
-  elif a.kind == nkSymChoice:
+  elif a.kind in {nkClosedSymChoice, nkOpenSymChoice}:
     # choose the generic proc with the proper number of type parameters.
     # XXX I think this could be improved by reusing sigmatch.ParamTypesMatch.
     # It's good enough for now.
-    result = newNodeI(nkSymChoice, n.info)
+    result = newNodeI(a.kind, n.info)
     for i in countup(0, len(a)-1): 
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter, skIterator}: 
@@ -144,8 +144,9 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:
           result.add(explicitGenericSym(c, n, candidate))
-    # get rid of nkSymChoice if not ambiguous:
+    # get rid of nkClosedSymChoice if not ambiguous:
-    if result.len == 1: result = result[0]
+    if result.len == 1 and a.kind == nkClosedSymChoice:
+      result = result[0]
     # candidateCount != 1: return explicitGenericInstError(n)
   else:
     result = explicitGenericInstError(n)

compiler/semexprs.nim

@@ -58,7 +58,7 @@ proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
 
 proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
-  result = symChoice(c, n, s)
+  result = symChoice(c, n, s, scClosed)
   
 proc inlineConst(n: PNode, s: PSym): PNode {.inline.} =
   result = copyTree(s.ast)
@@ -72,7 +72,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     if sfProcVar notin s.flags and s.typ.callConv == ccDefault and
         smoduleId != c.module.id and smoduleId != c.friendModule.id: 
       LocalError(n.info, errXCannotBePassedToProcVar, s.name.s)
-    result = symChoice(c, n, s)
+    result = symChoice(c, n, s, scClosed)
     if result.kind == nkSym:
       markIndirect(c, result.sym)
       if isGenericRoutine(result.sym):
@@ -182,6 +182,9 @@ proc isCastable(dst, src: PType): bool =
         (skipTypes(dst, abstractInst).kind in IntegralTypes) or
         (skipTypes(src, abstractInst).kind in IntegralTypes)
   
+proc isSymChoice(n: PNode): bool {.inline.} =
+  result = n.kind in {nkClosedSymChoice, nkOpenSymChoice}
+
 proc semConv(c: PContext, n: PNode, s: PSym): PNode = 
   if sonsLen(n) != 2: 
     LocalError(n.info, errConvNeedsOneArg)
@@ -191,7 +194,7 @@ proc semConv(c: PContext, n: PNode, s: PSym): PNode =
   addSon(result, copyTree(n.sons[0]))
   addSon(result, semExprWithType(c, n.sons[1]))
   var op = result.sons[1]
-  if op.kind != nkSymChoice: 
+  if not isSymChoice(op):
     checkConvertible(result.info, result.typ, op.typ)
   else: 
     for i in countup(0, sonsLen(op) - 1):
@@ -830,7 +833,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   ## returns nil if it's not a built-in field access
   checkSonsLen(n, 2)
   # early exit for this; see tests/compile/tbindoverload.nim:
-  if n.sons[1].kind == nkSymChoice: return
+  if isSymChoice(n.sons[1]): return
 
   var s = qualifiedLookup(c, n, {checkAmbiguity, checkUndeclared})
   if s != nil:
@@ -910,7 +913,7 @@ proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # in Nimrod. We first allow types in the semantic checking.
   result = builtinFieldAccess(c, n, flags)
   if result == nil:
-    if n.sons[1].kind == nkSymChoice: 
+    if isSymChoice(n.sons[1]):
       result = newNodeI(nkDotCall, n.info)
       addSon(result, n.sons[1])
       addSon(result, copyTree(n[0]))
@@ -1222,6 +1225,7 @@ proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
 proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = 
   # this is a hotspot in the compiler!
+  # DON'T forget to update ast.SpecialSemMagics if you add a magic here!
   result = n
   case s.magic # magics that need special treatment
   of mDefined: result = semDefined(c, setMs(n, s), false)
@@ -1518,8 +1522,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       else:
         #liMessage(n.info, warnUser, renderTree(n));
         result = semIndirectOp(c, n, flags)
-    elif n.sons[0].kind == nkSymChoice or n[0].kind == nkBracketExpr and 
+    elif isSymChoice(n.sons[0]) or n[0].kind == nkBracketExpr and 
-        n[0][0].kind == nkSymChoice:
+        isSymChoice(n[0][0]):
       result = semDirectOp(c, n, flags)
     else:
       result = semIndirectOp(c, n, flags)
@@ -1576,7 +1580,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     checkMinSonsLen(n, 2)
   of nkTableConstr:
     result = semTableConstr(c, n)
-  of nkSymChoice:
+  of nkClosedSymChoice, nkOpenSymChoice:
     LocalError(n.info, errExprXAmbiguous, renderTree(n, {renderNoComments}))
     # error correction: Pick first element:
     result = n.sons[0]

compiler/semgnrc.nim

@@ -13,7 +13,7 @@
 # A problem is that it cannot be detected if the symbol is introduced
 # as in ``var x = ...`` or used because macros/templates can hide this!
 # So we have to eval templates/macros right here so that symbol
-# lookup can be accurate.
+# lookup can be accurate. XXX But this can only be done for immediate macros!
 
 # included from sem.nim
 
@@ -47,8 +47,8 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym): PNode =
     # Introduced in this pass! Leave it as an identifier.
     result = n
   of skProc, skMethod, skIterator, skConverter: 
-    result = symChoice(c, n, s)
+    result = symChoice(c, n, s, scOpen)
-  of skTemplate: 
+  of skTemplate:
     result = semTemplateExpr(c, n, s, false)
   of skMacro: 
     result = semMacroExpr(c, n, s, false)
@@ -75,7 +75,8 @@ proc semGenericStmt(c: PContext, n: PNode,
       if withinBind in flags:
         localError(n.info, errUndeclaredIdentifier, n.ident.s)
     else:
-      if withinBind in flags or s.id in toBind: result = symChoice(c, n, s)
+      if withinBind in flags or s.id in toBind:
+        result = symChoice(c, n, s, scClosed)
       else: result = semGenericStmtSymbol(c, n, s)
   of nkDotExpr:
     var s = QualifiedLookUp(c, n, {})
@@ -104,7 +105,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam: 
         # Leave it as an identifier.
       of skProc, skMethod, skIterator, skConverter: 
-        result.sons[0] = symChoice(c, n.sons[0], s)
+        result.sons[0] = symChoice(c, n.sons[0], s, scOpen)
         first = 1
       of skGenericParam: 
         result.sons[0] = newSymNode(s, n.sons[0].info)
@@ -114,7 +115,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         if (s.typ != nil) and (s.typ.kind != tyGenericParam): 
           result.sons[0] = newSymNode(s, n.sons[0].info)
           first = 1
-      else: 
+      else:
         result.sons[0] = newSymNode(s, n.sons[0].info)
         first = 1
     for i in countup(first, sonsLen(result) - 1): 

compiler/semmagic.nim

@@ -49,16 +49,22 @@ proc semBindSym(c: PContext, n: PNode): PNode =
   result = copyNode(n)
   result.add(n.sons[0])
   
-  let sl = c.semConstExpr(c, n.sons[1])
+  let sl = semConstExpr(c, n.sons[1])
   if sl.kind notin {nkStrLit, nkRStrLit, nkTripleStrLit}: 
-    LocalError(n.info, errStringLiteralExpected)
+    LocalError(n.sons[1].info, errStringLiteralExpected)
+    return errorNode(c, n)
+  
+  let isMixin = semConstExpr(c, n.sons[2])
+  if isMixin.kind != nkIntLit or isMixin.intVal < 0 or
+      isMixin.intVal > high(TSymChoiceRule).int:
+    LocalError(n.sons[2].info, errConstExprExpected)
     return errorNode(c, n)
   
   let id = newIdentNode(getIdent(sl.strVal), n.info)
   let s = QualifiedLookUp(c, id)
   if s != nil:
     # we need to mark all symbols:
-    let sc = symChoice(c, id, s)
+    var sc = symChoice(c, id, s, TSymChoiceRule(isMixin.intVal))
     result.add(sc)
   else:
     LocalError(n.sons[1].info, errUndeclaredIdentifier, sl.strVal)

compiler/semtempl.nim

@@ -40,8 +40,12 @@ proc symBinding(n: PNode): TSymBinding =
       of wInject: return spInject
       else: nil
 
-proc symChoice(c: PContext, n: PNode, s: PSym): PNode = 
+type
-  var 
+  TSymChoiceRule = enum
+    scClosed, scOpen, scForceOpen
+
+proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
+  var
     a: PSym
     o: TOverloadIter
   var i = 0
@@ -50,17 +54,17 @@ proc symChoice(c: PContext, n: PNode, s: PSym): PNode =
     a = nextOverloadIter(o, c, n)
     inc(i)
     if i > 1: break
-  if i <= 1:
+  if i <= 1 and r != scForceOpen:
     # XXX this makes more sense but breaks bootstrapping for now:
     # (s.kind notin routineKinds or s.magic != mNone):
-    # for some reason 'nextTry' is copied and considered as a candidate in
+    # for instance 'nextTry' is both in tables.nim and astalgo.nim ...
-    # tables.nim
     result = newSymNode(s, n.info)
     markUsed(n, s)
   else:
     # semantic checking requires a type; ``fitNode`` deals with it
     # appropriately
-    result = newNodeIT(nkSymChoice, n.info, newTypeS(tyNone, c))
+    let kind = if r == scClosed: nkClosedSymChoice else: nkOpenSymChoice
+    result = newNodeIT(kind, n.info, newTypeS(tyNone, c))
     a = initOverloadIter(o, c, n)
     while a != nil:
       incl(a.flags, sfUsed)
@@ -78,7 +82,7 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var TIntSet): PNode =
     let s = QualifiedLookUp(c, a)
     if s != nil:
       # we need to mark all symbols:
-      let sc = symChoice(c, n, s)
+      let sc = symChoice(c, n, s, scClosed)
       if sc.kind == nkSym:
         toBind.incl(sc.sym.id)
       else:
@@ -178,7 +182,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         incl(s.flags, sfUsed)
         result = newSymNode(s, n.info)
       elif Contains(c.toBind, s.id):
-        result = symChoice(c.c, n, s)
+        result = symChoice(c.c, n, s, scClosed)
       elif s.owner == c.owner:
         InternalAssert sfGenSym in s.flags
         incl(s.flags, sfUsed)
@@ -294,7 +298,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     if n.kind == nkDotExpr or n.kind == nkAccQuoted:
       let s = QualifiedLookUp(c.c, n, {})
       if s != nil and Contains(c.toBind, s.id):
-        return symChoice(c.c, n, s)
+        return symChoice(c.c, n, s, scClosed)
     result = n
     for i in countup(0, sonsLen(n) - 1):
       result.sons[i] = semTemplBody(c, n.sons[i])
@@ -308,7 +312,7 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
       if s.owner == c.owner and s.kind == skParam:
         result = newSymNode(s, n.info)
       elif Contains(c.toBind, s.id):
-        result = symChoice(c.c, n, s)
+        result = symChoice(c.c, n, s, scClosed)
   of nkBind:
     result = semTemplBodyDirty(c, n.sons[0])
   of nkBindStmt:
@@ -321,7 +325,7 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
     if n.kind == nkDotExpr or n.kind == nkAccQuoted:
       let s = QualifiedLookUp(c.c, n, {})
       if s != nil and Contains(c.toBind, s.id):
-        return symChoice(c.c, n, s)
+        return symChoice(c.c, n, s, scClosed)
     result = n
     for i in countup(0, sonsLen(n) - 1):
       result.sons[i] = semTemplBodyDirty(c, n.sons[i])

compiler/sigmatch.nim

@@ -703,7 +703,7 @@ proc ParamTypesMatchAux(c: PContext, m: var TCandidate, f, a: PType,
 
 proc ParamTypesMatch(c: PContext, m: var TCandidate, f, a: PType, 
                      arg, argOrig: PNode): PNode = 
-  if arg == nil or arg.kind != nkSymChoice: 
+  if arg == nil or arg.kind != nkClosedSymChoice:
     result = ParamTypesMatchAux(c, m, f, a, arg, argOrig)
   else: 
     # CAUTION: The order depends on the used hashing scheme. Thus it is

compiler/suggest.nim

@@ -85,7 +85,7 @@ proc suggestObject(c: PContext, n: PNode, outputs: var int) =
 
 proc nameFits(c: PContext, s: PSym, n: PNode): bool = 
   var op = n.sons[0]
-  if op.kind == nkSymChoice: op = op.sons[0]
+  if op.kind in {nkOpenSymChoice, nkClosedSymChoice}: op = op.sons[0]
   var opr: PIdent
   case op.kind
   of nkSym: opr = op.sym.name

compiler/trees.nim

@@ -143,7 +143,8 @@ proc SwapOperands*(op: PNode) =
 proc IsRange*(n: PNode): bool {.inline.} = 
   if n.kind == nkInfix:
     if n[0].kind == nkIdent and n[0].ident.id == ord(wDotDot) or
-        n[0].kind == nkSymChoice and n[0][1].sym.name.id == ord(wDotDot):
+        n[0].kind in {nkClosedSymChoice, nkOpenSymChoice} and 
+        n[0][1].sym.name.id == ord(wDotDot):
       result = true
 
 proc whichPragma*(n: PNode): TSpecialWord = 

compiler/types.nim

@@ -117,7 +117,7 @@ proc isCompatibleToCString(a: PType): bool =
       result = true
   
 proc getProcHeader(sym: PSym): string = 
-  result = sym.name.s & '('
+  result = sym.owner.name.s & '.' & sym.name.s & '('
   var n = sym.typ.n
   for i in countup(1, sonsLen(n) - 1): 
     var p = n.sons[i]

doc/manual.txt

@@ -2362,6 +2362,19 @@ notation. (Thus an operator can have more than two parameters):
 
   assert `*+`(3, 4, 6) == `*`(a, `+`(b, c))
 
+
+Nonoverloadable builtins
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following builtin procs cannot be overloaded for reasons of implementation
+simplicity (they require specialized semantic checking)::
+
+  defined, definedInScope, compiles, low, high, sizeOf, 
+  is, of, echo, shallowCopy, getAst
+
+Thus they act more like keywords than like ordinary identifiers; unlike a 
+keyword however, a redefinition may `shadow`:id: the definition in 
+the ``system`` module.
 
 
 Var parameters

lib/core/macros.nim

@@ -27,7 +27,10 @@ type
     nnkBracket, nnkBracketExpr, nnkPragmaExpr, nnkRange, 
     nnkDotExpr, nnkCheckedFieldExpr, nnkDerefExpr, nnkIfExpr, 
     nnkElifExpr, nnkElseExpr, nnkLambda, nnkDo, nnkAccQuoted, 
-    nnkTableConstr, nnkBind, nnkSymChoice, nnkHiddenStdConv, 
+    nnkTableConstr, nnkBind,
+    nnkClosedSymChoice,
+    nnkOpenSymChoice,
+    nnkHiddenStdConv,
     nnkHiddenSubConv, nnkHiddenCallConv, nnkConv, nnkCast, nnkStaticExpr,
     nnkAddr, nnkHiddenAddr, nnkHiddenDeref, nnkObjDownConv, 
     nnkObjUpConv, nnkChckRangeF, nnkChckRange64, nnkChckRange, 
@@ -186,12 +189,29 @@ proc newIdentNode*(i: string): PNimrodNode {.compileTime.} =
   result = newNimNode(nnkIdent)
   result.ident = !i
 
-proc bindSym*(ident: string): PNimrodNode {.magic: "NBindSym".}
+type
+  TBindSymRule* = enum   ## specifies how ``bindSym`` behaves
+    brClosed,            ## only the symbols in current scope are bound
+    brOpen,              ## open wrt overloaded symbols, but may be a single
+                         ## symbol if not ambiguous (the rules match that of
+                         ## binding in generics)
+    brForceOpen          ## same as brOpen, but it will always be open even
+                         ## if not ambiguous (this cannot be achieved with
+                         ## any other means in the language currently)
+
+proc bindSym*(ident: string, rule: TBindSymRule = brClosed): PNimrodNode {.
+              magic: "NBindSym".}
   ## creates a node that binds `ident` to a symbol node. The bound symbol
-  ## needs to be predeclared in a ``bind`` statement!
+  ## may be an overloaded symbol.
+  ## If ``rule == brClosed`` either an ``nkClosedSymChoice`` tree is
+  ## returned or ``nkSym`` if the symbol is not ambiguous.
+  ## If ``rule == brOpen`` either an ``nkOpenSymChoice`` tree is
+  ## returned or ``nkSym`` if the symbol is not ambiguous.
+  ## If ``rule == brForceOpen`` always an ``nkOpenSymChoice`` tree is
+  ## returned even if the symbol is not ambiguous.
 
 proc toStrLit*(n: PNimrodNode): PNimrodNode {.compileTime.} =
-  ## converts the AST `n` to the concrete Nimrod code and wraps that 
+  ## converts the AST `n` to the concrete Nimrod code and wraps that
   ## in a string literal node
   return newStrLitNode(repr(n))
 

todo.txt

@@ -1,6 +1,10 @@
 version 0.9.0
 =============
 
+- make 'bind' default for templates and introduce 'mixin'
+- introduce 'callsite' magic and make macros and templates the same
+- implement generic templates/macros
+
 - implement "closure tuple consists of a single 'ref'" optimization
 - implement for loop transformation for first class iterators
 
@@ -29,8 +33,6 @@ Bugs
 version 0.9.XX
 ==============
 
-- make 'bind' default for templates and introduce 'mixin'
-- distinguish between open and closed nkSymChoice
 - JS gen:
   - fix exception handling
 
@@ -44,7 +46,6 @@ version 0.9.XX
 - ``hoist`` pragma for loop hoisting
 - document destructors; don't work yet when used as expression
 - make use of ``tyIter`` to fix the implicit items/pairs issue
-- introduce 'callsite' magic and make macros and templates the same
 - better support for macros that rewrite procs
 - macros need access to types and symbols
 - document nimdoc properly finally
@@ -56,7 +57,6 @@ version 0.9.XX
 - proc specialization in the code gen for write barrier specialization
 - tlastmod returns wrong results on BSD (Linux, MacOS X: works)
 - nested tuple unpacking; tuple unpacking in non-var-context
-- 'nimrod def': does not always work?
 - test branch coverage
 - make pegs support a compile-time option and make c2nim use regexes instead
   per default?