iterable[T] (#17196)

* fix failing test toSeq in manual which now works
* changelog
* reject proc fn(a: iterable)
* add iterable to spec
* remove MCS/UFCS limitation that now works

changelog.md

@@ -302,6 +302,10 @@
 - `nim e` now accepts arbitrary file extensions for the nimscript file,
   although `.nims` is still the preferred extension in general.
 
+- Added `iterable[T]` type class to match called iterators, which enables writing:
+  `template fn(a: iterable)` instead of `template fn(a: untyped)`
+
+
 ## Compiler changes
 
 - Added `--declaredlocs` to show symbol declaration location in messages.
@@ -341,6 +345,7 @@
 - `--hint:CC` now goes to stderr (like all other hints) instead of stdout.
 
 
+
 ## Tool changes
 
 - The rst parser now supports markdown table syntax.

compiler/ast.nim

@@ -440,10 +440,12 @@ type
 
     tyVoid
       # now different from tyEmpty, hurray!
+    tyIterable
 
 static:
   # remind us when TTypeKind stops to fit in a single 64-bit word
-  assert TTypeKind.high.ord <= 63
+  # assert TTypeKind.high.ord <= 63
+  discard
 
 const
   tyPureObject* = tyTuple
@@ -664,7 +666,7 @@ type
     mDefault, mUnown, mIsolate, mAccessEnv, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
-    mOrdinal,
+    mOrdinal, mIterableType,
     mInt, mInt8, mInt16, mInt32, mInt64,
     mUInt, mUInt8, mUInt16, mUInt32, mUInt64,
     mFloat, mFloat32, mFloat64, mFloat128,

compiler/condsyms.nim

@@ -132,3 +132,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasSpellSuggest")
   defineSymbol("nimHasCustomLiterals")
   defineSymbol("nimHasUnifiedTuple")
+  defineSymbol("nimHasIterable")

compiler/jsgen.nim

@@ -211,7 +211,7 @@ proc mapType(typ: PType): TJSTypeKind =
     else: result = etyNone
   of tyProc: result = etyProc
   of tyCString: result = etyString
-  of tyConcept: doAssert false
+  of tyConcept, tyIterable: doAssert false
 
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   result = mapType(typ)

compiler/liftdestructors.nim

@@ -908,7 +908,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of tyOrdinal, tyRange, tyInferred,
      tyGenericInst, tyAlias, tySink:
     fillBody(c, lastSon(t), body, x, y)
-  of tyConcept: doAssert false
+  of tyConcept, tyIterable: doAssert false
 
 proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
                             kind: TTypeAttachedOp; info: TLineInfo;

compiler/linter.nim

@@ -40,7 +40,7 @@ proc beautifyName(s: string, k: TSymKind): string =
              "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
              "cshort", "cu", "nil", "typedesc", "auto", "any",
              "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
-             "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
+             "untyped", "typed", "static", "sink", "lent", "type", "owned", "iterable"]:
       result.add s[i]
     else:
       result.add toUpperAscii(s[i])

compiler/semcall.nim

@@ -62,7 +62,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
                        best, alt: var TCandidate,
                        errors: var CandidateErrors,
                        diagnosticsFlag: bool,
-                       errorsEnabled: bool) =
+                       errorsEnabled: bool, flags: TExprFlags) =
   var o: TOverloadIter
   var sym = initOverloadIter(o, c, headSymbol)
   var scope = o.lastOverloadScope
@@ -95,7 +95,11 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
       matches(c, n, orig, z)
       if z.state == csMatch:
         # little hack so that iterators are preferred over everything else:
-        if sym.kind == skIterator: inc(z.exactMatches, 200)
+        if sym.kind == skIterator:
+          if not (efWantIterator notin flags and efWantIterable in flags):
+            inc(z.exactMatches, 200)
+          else:
+            dec(z.exactMatches, 200)
         case best.state
         of csEmpty, csNoMatch: best = z
         of csMatch:
@@ -356,7 +360,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
   template pickBest(headSymbol) =
     pickBestCandidate(c, headSymbol, n, orig, initialBinding,
                       filter, result, alt, errors, efExplain in flags,
-                      errorsEnabled)
+                      errorsEnabled, flags)
   pickBest(f)
 
   let overloadsState = result.state

compiler/semdata.nim

@@ -54,7 +54,7 @@ type
     inst*: PInstantiation
 
   TExprFlag* = enum
-    efLValue, efWantIterator, efInTypeof,
+    efLValue, efWantIterator, efWantIterable, efInTypeof,
     efNeedStatic,
       # Use this in contexts where a static value is mandatory
     efPreferStatic,

compiler/semexprs.nim

@@ -853,7 +853,7 @@ proc semStaticExpr(c: PContext, n: PNode): PNode =
 
 proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
-  if flags*{efInTypeof, efWantIterator} != {}:
+  if flags*{efInTypeof, efWantIterator, efWantIterable} != {}:
     # consider: 'for x in pReturningArray()' --> we don't want the restriction
     # to 'skIterator' anymore; skIterator is preferred in sigmatch already
     # for typeof support.
@@ -877,6 +877,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
         # error correction, prevents endless for loop elimination in transf.
         # See bug #2051:
         result[0] = newSymNode(errorSym(c, n))
+      elif callee.kind == skIterator:
+        if efWantIterable in flags:
+          let typ = newTypeS(tyIterable, c)
+          rawAddSon(typ, result.typ)
+          result.typ = typ
 
 proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode
 
@@ -1364,7 +1369,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
     onUse(n[1].info, s)
     return
 
-  n[0] = semExprWithType(c, n[0], flags+{efDetermineType})
+  n[0] = semExprWithType(c, n[0], flags+{efDetermineType, efWantIterable})
   #restoreOldStyleType(n[0])
   var i = considerQuotedIdent(c, n[1], n)
   var ty = n[0].typ

compiler/semtypes.nim

@@ -356,6 +356,15 @@ proc semArray(c: PContext, n: PNode, prev: PType): PType =
     localError(c.config, n.info, errArrayExpectsTwoTypeParams)
     result = newOrPrevType(tyError, prev, c)
 
+proc semIterableType(c: PContext, n: PNode, prev: PType): PType =
+  result = newOrPrevType(tyIterable, prev, c)
+  if n.len == 2:
+    let base = semTypeNode(c, n[1], nil)
+    addSonSkipIntLit(result, base, c.idgen)
+  else:
+    localError(c.config, n.info, errXExpectsOneTypeParam % "iterable")
+    result = newOrPrevType(tyError, prev, c)
+
 proc semOrdinal(c: PContext, n: PNode, prev: PType): PType =
   result = newOrPrevType(tyOrdinal, prev, c)
   if n.len == 2:
@@ -1844,6 +1853,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mRange: result = semRange(c, n, prev)
     of mSet: result = semSet(c, n, prev)
     of mOrdinal: result = semOrdinal(c, n, prev)
+    of mIterableType: result = semIterableType(c, n, prev)
     of mSeq:
       result = semContainer(c, n, tySequence, "seq", prev)
       if optSeqDestructors in c.config.globalOptions:
@@ -2067,6 +2077,9 @@ proc processMagicType(c: PContext, m: PSym) =
   of mOrdinal:
     setMagicIntegral(c.config, m, tyOrdinal, szUncomputedSize)
     rawAddSon(m.typ, newTypeS(tyNone, c))
+  of mIterableType:
+    setMagicIntegral(c.config, m, tyIterable, 0)
+    rawAddSon(m.typ, newTypeS(tyNone, c))
   of mPNimrodNode:
     incl m.typ.flags, tfTriggersCompileTime
     incl m.typ.flags, tfCheckedForDestructor

compiler/sigmatch.nim

@@ -1119,6 +1119,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
         return if x >= isGeneric: isGeneric else: x
     return isNone
 
+  of tyIterable:
+    if f.kind != tyIterable: return isNone
   of tyNot:
     case f.kind
     of tyNot:
@@ -1421,6 +1423,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
   of tyAlias, tySink:
     result = typeRel(c, lastSon(f), a, flags)
 
+  of tyIterable:
+    if a.kind == tyIterable:
+      if f.len == 1:
+        result = typeRel(c, lastSon(f), lastSon(a), flags)
+      else:
+        # f.len = 3, for some reason
+        result = isGeneric
+    else:
+      result = isNone
   of tyGenericInst:
     var prev = PType(idTableGet(c.bindings, f))
     let origF = f
@@ -2270,14 +2281,18 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode =
     # a.typ == nil is valid
     result = a
   elif a.typ.isNil:
-    # XXX This is unsound! 'formal' can differ from overloaded routine to
+    if formal.kind == tyIterable:
-    # overloaded routine!
+      let flags = {efDetermineType, efAllowStmt, efWantIterator, efWantIterable}
-    let flags = {efDetermineType, efAllowStmt}
+      result = c.semOperand(c, a, flags)
-                #if formal.kind == tyIter: {efDetermineType, efWantIterator}
+    else:
-                #else: {efDetermineType, efAllowStmt}
+      # XXX This is unsound! 'formal' can differ from overloaded routine to
-                #elif formal.kind == tyTyped: {efDetermineType, efWantStmt}
+      # overloaded routine!
-                #else: {efDetermineType}
+      let flags = {efDetermineType, efAllowStmt}
-    result = c.semOperand(c, a, flags)
+                  #if formal.kind == tyIterable: {efDetermineType, efWantIterator}
+                  #else: {efDetermineType, efAllowStmt}
+                  #elif formal.kind == tyTyped: {efDetermineType, efWantStmt}
+                  #else: {efDetermineType}
+      result = c.semOperand(c, a, flags)
   else:
     result = a
     considerGenSyms(c, result)

compiler/typeallowed.nim

@@ -97,6 +97,9 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       result = nil
   of tyUntyped, tyTyped:
     if kind notin {skParam, skResult} or taNoUntyped in flags: result = t
+  of tyIterable:
+    if kind notin {skParam} or taNoUntyped in flags: result = t
+      # tyIterable is only for templates and macros.
   of tyStatic:
     if kind notin {skParam}: result = t
   of tyVoid:

compiler/types.nim

@@ -460,8 +460,8 @@ const
     "lent ", "varargs[$1]", "UncheckedArray[$1]", "Error Type",
     "BuiltInTypeClass", "UserTypeClass",
     "UserTypeClassInst", "CompositeTypeClass", "inferred",
-    "and", "or", "not", "any", "static", "TypeFromExpr", "out ",
+    "and", "or", "not", "any", "static", "TypeFromExpr", "concept", # xxx bugfix
-    "void"]
+    "void", "iterable"]
 
 const preferToResolveSymbols = {preferName, preferTypeName, preferModuleInfo,
   preferGenericArg, preferResolved, preferMixed}
@@ -645,6 +645,11 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     of tyDistinct:
       result = "distinct " & typeToString(t[0],
         if prefer == preferModuleInfo: preferModuleInfo else: preferTypeName)
+    of tyIterable:
+      # xxx factor this pattern
+      result = "iterable"
+      if t.len > 0:
+        result &= "[" & typeToString(t[0]) & ']'
     of tyTuple:
       # we iterate over t.sons here, because t.n may be nil
       if t.n != nil:
@@ -1191,7 +1196,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
     result = sameTypeOrNilAux(a[0], b[0], c) and
         sameValue(a.n[0], b.n[0]) and
         sameValue(a.n[1], b.n[1])
-  of tyGenericInst, tyAlias, tyInferred:
+  of tyGenericInst, tyAlias, tyInferred, tyIterable:
     cycleCheck()
     result = sameTypeAux(a.lastSon, b.lastSon, c)
   of tyNone: result = false

compiler/vmdeps.nim

@@ -287,6 +287,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo;
   of tyAnd: result = mapTypeToBracket("and", mAnd, t, info)
   of tyOr: result = mapTypeToBracket("or", mOr, t, info)
   of tyNot: result = mapTypeToBracket("not", mNot, t, info)
+  of tyIterable: result = mapTypeToBracket("iterable", mIterableType, t, info)
   of tyAnything: result = atomicType("anything", mNone)
   of tyInferred: assert false
   of tyStatic, tyFromExpr:

doc/manual.rst

@@ -2645,6 +2645,28 @@ Varargs matching
 
 See `Varargs <#types-varargs>`_.
 
+iterable
+--------
+
+A called `iterator` yielding type `T` can be passed to a template or macro via
+a parameter typed as `untyped` (for unresolved expressions) or the type class
+`iterable` or `iterable[T]` (after type checking and overload resolution).
+
+.. code-block:: nim
+  iterator iota(n: int): int =
+    for i in 0..<n: yield i
+
+  template toSeq2[T](a: iterable[T]): seq[T] =
+    var ret: seq[T]
+    assert a.typeof is T
+    for ai in a: ret.add ai
+    ret
+
+  assert iota(3).toSeq2 == @[0, 1, 2]
+  assert toSeq2(5..7) == @[5, 6, 7]
+  assert not compiles(toSeq2(@[1,2])) # seq[int] is not an iterable
+  assert toSeq2(items(@[1,2])) == @[1, 2] # but items(@[1,2]) is
+
 
 Statements and expressions
 ==========================
@@ -4351,6 +4373,9 @@ would.  For example:
   for i in toItr(recCountDown(6)): # Emits: 6 5 4 3 2 1
     echo i
 
+
+See also see `iterable <#overloading-resolution-iterable>`_ for passing iterators to templates and macros.
+
 Converters
 ==========
 
@@ -5591,24 +5616,6 @@ is used to invoke templates/macros:
   unknownIdentifier.declareVar
 
 
-Another common example is this:
-
-.. code-block:: nim
-    :test: "nim c $1"
-    :status: 1
-
-  from std/sequtils import toSeq
-
-  iterator something: string =
-    yield "Hello"
-    yield "World"
-
-  var info = something().toSeq
-
-The problem here is that the compiler already decided that `something()` as
-an iterator is not callable in this context before `toSeq` gets its
-chance to convert it into a sequence.
-
 It is also not possible to use fully qualified identifiers with module
 symbol in method call syntax. The order in which the dot operator
 binds to symbols prohibits this.

lib/system.nim

@@ -123,6 +123,10 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.}
   ##     # Do here programmer friendly expensive sanity checks.
   ##   # Put here the normal code
 
+when defined(nimHasIterable):
+  type
+    iterable*[T] {.magic: IterableType.}  ## Represents an expression that yields `T`
+
 when defined(nimHashOrdinalFixed):
   type
     Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer,

testament/lib/stdtest/testutils.nim

@@ -79,3 +79,9 @@ template whenVMorJs*(bodyIf, bodyElse) =
   else:
     when defined(js): bodyIf
     else: bodyElse
+
+template accept*(a) =
+  doAssert compiles(a)
+
+template reject*(a) =
+  doAssert not compiles(a)

tests/types/titerable.nim

@@ -0,0 +1,143 @@
+discard """
+  targets: "c js"
+"""
+
+from stdtest/testutils import accept, reject, whenVMorJs
+
+# toSeq-like templates
+
+template toSeq2(a: iterable): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template toSeq3(a: iterable[string]): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template toSeq4[T](a: iterable[T]): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template toSeq5[T: SomeInteger](a: iterable[T]): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template toSeq6(a: iterable[int | float]): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template toSeq7(a: iterable[seq]): auto =
+  var ret: seq[typeof(a)]
+  for ai in a: ret.add ai
+  ret
+
+template fn7b(a: untyped) = discard
+template fn7c(a: typed) = discard
+template fn7d(a: auto) = discard
+template fn7e[T](a: T) = discard
+
+template fn8a(a: iterable) = discard
+template fn8b[T](a: iterable[T]) = discard
+template fn8c(a: iterable[int]) = discard
+
+template bad1 =
+  template fn4(a: int, b: iterable[float, int]) =
+    discard
+
+template bad2 =
+  proc fn4(a: iterable) = discard
+
+template bad3 =
+  proc fn4(a: iterable[int]) = discard
+
+template good4 =
+  template fn1(a: iterable) = discard
+  template fn2(a: iterable[int]) = discard
+
+# iterators
+iterator iota(n: int): auto =
+  for i in 0..<n: yield i
+
+iterator repeat[T](a: T, n: int): T =
+  for i in 0..<n:
+    yield a
+
+iterator myiter(n: int): auto =
+  for i in 0..<n: yield $(i*2)
+
+when not defined(js):
+  iterator iotaClosure(n: int): auto {.closure.} =
+    for i in 0..<n: yield i
+
+template main() =
+  let expected1 = @[0, 1, 2]
+  let expected2 = @["0", "2"]
+
+  doAssert toSeq2(myiter(2)) == expected2
+  doAssert toSeq2(iota(3)) == expected1
+  doAssert toSeq2(1.1.repeat(2)) == @[1.1, 1.1]
+
+  whenVMorJs: discard
+  do:
+    doAssert toSeq2(iotaClosure(3)) == expected1
+
+  when true:
+    # MCS/UFCS
+    doAssert iota(3).toSeq2() == expected1
+
+  doAssert toSeq3(myiter(2)) == expected2
+  accept toSeq3(myiter(2))
+  reject toSeq3(iota(3))
+
+  doAssert toSeq4(iota(3)) == expected1
+  doAssert toSeq4(myiter(2)) == expected2
+  
+  doAssert toSeq4(0..2) == expected1
+  doAssert toSeq4(items(0..2)) == expected1
+  doAssert toSeq4(items(@[0,1,2])) == expected1
+  reject toSeq4(@[0,1,2])
+  reject toSeq4(13)
+
+  block:
+    accept fn8a(iota(3))
+    accept fn7b(iota(3))
+    reject fn7c(iota(3))
+    reject fn7d(iota(3))
+    reject fn7e(iota(3))
+
+  block:
+    fn8a(iota(3))
+    reject fn8a(123)
+    reject fn8c(123)
+    reject fn8c(123.3)
+    accept fn8c(items(@[1,2]))
+
+  block:
+    # shows that iterable is more restrictive than untyped
+    reject fn8a(nonexistant)
+    accept fn7b(nonexistant)
+    reject fn7c(nonexistant)
+    reject fn7d(nonexistant)
+    reject fn7e(nonexistant)
+
+  doAssert toSeq5(iota(3)) == expected1
+  reject toSeq5(myiter(2))
+
+  doAssert toSeq6(iota(3)) == expected1
+  reject toSeq6(myiter(2))
+
+  doAssert toSeq2("abc".repeat(3)) == @["abc", "abc", "abc"]
+  doAssert toSeq2(@['x', 'y'].repeat(2)) == @[@['x', 'y'], @['x', 'y']]
+
+  reject bad1
+  reject bad2
+  reject bad3
+  accept good4
+
+static: main()
+main()