idetools improvements; preparation of first class iterators; fixes #183

compiler/ast.nim

@@ -200,7 +200,9 @@ type
     nkEnumTy,             # enum body
     nkEnumFieldDef,       # `ident = expr` in an enumeration
     nkReturnToken,        # token used for interpretation
-    nkClosure             # (prc, env)-pair (internally used for code gen)
+    nkClosure,            # (prc, env)-pair (internally used for code gen)
+    nkGotoState,          # used for the state machine (for iterators)
+    nkState               # give a label to a code section (for iterators)
   TNodeKinds* = set[TNodeKind]
 
 type

compiler/ccgstmts.nim

@@ -92,6 +92,25 @@ template preserveBreakIdx(body: stmt): stmt =
   body
   p.breakIdx = oldBreakIdx
 
+proc genState(p: BProc, n: PNode) =
+  internalAssert n.len == 2 and n.sons[0].kind == nkIntLit
+  let idx = n.sons[0].intVal
+  lineCg(p, cpsStmts, "STATE$1: ;$n", [idx.toRope])
+  genStmts(p, n.sons[1])
+
+proc genGotoState(p: BProc, n: PNode) =
+  # we resist the temptation to translate it into duff's device as it later
+  # will be translated into computed gotos anyway for GCC at least:
+  # switch (x.state) {
+  #   case 0: goto STATE0;
+  # ...
+  var a: TLoc
+  initLocExpr(p, n.sons[0], a)
+  lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
+  for i in 0 .. lastOrd(n.sons[0].typ):
+    lineF(p, cpsStmts, "case $1: goto STATE$1;$n", [toRope(i)])
+  lineF(p, cpsStmts, "}$n", [])
+
 proc genSingleVar(p: BProc, a: PNode) =
   var v = a.sons[0].sym
   if sfCompileTime in v.flags: return
@@ -863,5 +882,7 @@ proc genStmts(p: BProc, t: PNode) =
         if prc.getBody.kind != nkEmpty or lfDynamicLib in prc.loc.flags: 
           genProc(p.module, prc)
   of nkParForStmt: genParForStmt(p, t)
+  of nkState: genState(p, t)
+  of nkGotoState: genGotoState(p, t)
   else: internalError(t.info, "genStmts(" & $t.kind & ')')
   

compiler/ccgtrav.nim

@@ -44,6 +44,8 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: PRope, n: PNode) =
     lineF(p, cpsStmts, "} $n")
   of nkSym:
     let field = n.sym
+    if field.loc.t == nil:
+      internalError(n.info, "genTraverseProc()")
     genTraverseProc(c, ropef("$1.$2", accessor, field.loc.r), field.loc.t)
   else: internalError(n.info, "genTraverseProc()")
 
@@ -73,11 +75,8 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: PRope, typ: PType) =
     if typ.n != nil: genTraverseProc(c, accessor, typ.n)
   of tyTuple:
     let typ = GetUniqueType(typ)
-    if typ.n != nil:
+    for i in countup(0, sonsLen(typ) - 1):
-      genTraverseProc(c, accessor, typ.n)
+      genTraverseProc(c, ropef("$1.Field$2", accessor, i.toRope), typ.sons[i])
-    else:
-      for i in countup(0, sonsLen(typ) - 1):
-        genTraverseProc(c, ropef("$1.Field$2", accessor, i.toRope), typ.sons[i])
   of tyRef, tyString, tySequence:
     lineCg(p, cpsStmts, c.visitorFrmt, accessor)
   of tyProc:

compiler/ecmasgen.nim

@@ -1468,6 +1468,8 @@ proc genStmt(p: var TProc, n: PNode, r: var TCompRes) =
     if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}: 
       var r2: TCompRes
       genSym(p, n.sons[namePos], r2)
+  of nkGotoState, nkState:
+    internalError(n.info, "first class iterators not implemented")
   else:
     genLineDir(p, n, r)
     gen(p, n, r)

compiler/importer.nim

@@ -37,7 +37,7 @@ proc checkModuleName*(n: PNode): string =
   var modulename = n.getModuleName
   result = findModule(modulename)
   if result.len == 0:
-    Fatal(n.info, errCannotOpenFile, modulename)
+    LocalError(n.info, errCannotOpenFile, modulename)
 
 proc rawImportSymbol(c: PContext, s: PSym) = 
   # This does not handle stubs, because otherwise loading on demand would be
@@ -109,19 +109,20 @@ proc evalImport(c: PContext, n: PNode): PNode =
   result = n
   for i in countup(0, sonsLen(n) - 1): 
     var f = checkModuleName(n.sons[i])
-    var m = gImportModule(f)
+    if f.len > 0:
-    if sfDeprecated in m.flags: 
+      var m = gImportModule(f)
-      Message(n.sons[i].info, warnDeprecated, m.name.s) 
+      if sfDeprecated in m.flags: 
-    # ``addDecl`` needs to be done before ``importAllSymbols``!
+        Message(n.sons[i].info, warnDeprecated, m.name.s) 
-    addDecl(c, m)             # add symbol to symbol table of module
+      # ``addDecl`` needs to be done before ``importAllSymbols``!
-    importAllSymbols(c, m)
+      addDecl(c, m)             # add symbol to symbol table of module
+      importAllSymbols(c, m)
 
 proc evalFrom(c: PContext, n: PNode): PNode = 
   result = n
   checkMinSonsLen(n, 2)
   var f = checkModuleName(n.sons[0])
-  var m = gImportModule(f)
+  if f.len > 0:
-  n.sons[0] = newSymNode(m)
+    var m = gImportModule(f)
-  addDecl(c, m)               # add symbol to symbol table of module
+    n.sons[0] = newSymNode(m)
-  for i in countup(1, sonsLen(n) - 1): importSymbol(c, n.sons[i], m)
+    addDecl(c, m)               # add symbol to symbol table of module
-  
+    for i in countup(1, sonsLen(n) - 1): importSymbol(c, n.sons[i], m)

compiler/lambdalifting.nim

@@ -8,7 +8,6 @@
 #
 
 # This include file implements lambda lifting for the transformator.
-# included from transf.nim
 
 import 
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, 
@@ -517,3 +516,30 @@ proc liftLambdas*(n: PNode): PNode =
   var s = n.sons[namePos].sym
   if gCmd == cmdCompileToEcmaScript: return s.getBody
   result = liftLambdas(s, s.getBody)
+
+proc transformIterator*(fn: PSym, body: PNode): PNode =
+  if body.kind == nkEmpty:
+    # ignore forward declaration:
+    result = body
+  # it(a, b) --> (it(a, b), createClosure())
+  # it(a, b) --> ?
+  discard """
+  let c = chain(f, g)
+  
+  for x in c: echo x
+  
+  iterator chain[S, T](a, b: *S->T, args: *S): T =
+    for x in a(args): yield x
+    for x in b(args): yield x
+  
+  # translated to:
+  
+  
+  
+  let c = chain( (f, newClosure(f)), (g, newClosure(g)), newClosure(chain))
+  
+  
+"""
+
+
+

compiler/msgs.nim

@@ -344,7 +344,7 @@ const
     warnLanguageXNotSupported: "language \'$1\' not supported [LanguageXNotSupported]", 
     warnCommentXIgnored: "comment \'$1\' ignored [CommentXIgnored]", 
     warnXisPassedToProcVar: "\'$1\' is passed to a procvar; deprecated [XisPassedToProcVar]", 
-    warnAnalysisLoophole: "thread analysis incomplete due to unkown call '$1' [AnalysisLoophole]",
+    warnAnalysisLoophole: "thread analysis incomplete due to unknown call '$1' [AnalysisLoophole]",
     warnDifferentHeaps: "possible inconsistency of thread local heaps [DifferentHeaps]",
     warnWriteToForeignHeap: "write to foreign heap [WriteToForeignHeap]",
     warnImplicitClosure: "implicit closure convention: '$1' [ImplicitClosure]",

compiler/parser.nim

@@ -275,6 +275,7 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
 
 proc dotExpr(p: var TParser, a: PNode): PNode =
   getTok(p)
+  var info = p.lex.getlineInfo
   optInd(p, a)
   case p.tok.tokType
   of tkType:
@@ -286,7 +287,7 @@ proc dotExpr(p: var TParser, a: PNode): PNode =
     getTok(p)
     addSon(result, a)
   else:
-    result = newNodeI(nkDotExpr, a.info)
+    result = newNodeI(nkDotExpr, info)
     addSon(result, a)
     addSon(result, parseSymbol(p))
 
@@ -758,9 +759,10 @@ proc parseExpr(p: var TParser): PNode =
   case p.tok.tokType:
   of tkIf: result = parseIfExpr(p, nkIfExpr)
   of tkWhen: result = parseIfExpr(p, nkWhenExpr)
-  of tkTry: result = parseTry(p)
-  of tkCase: result = parseCase(p)
   else: result = lowestExpr(p)
+  # XXX needs proper support:
+  #of tkCase: result = parseCase(p)
+  #of tkTry: result = parseTry(p)
 
 proc primary(p: var TParser, skipSuffix = false): PNode = 
   # prefix operator?

compiler/semstmts.nim

@@ -841,6 +841,8 @@ proc semIterator(c: PContext, n: PNode): PNode =
   var t = s.typ
   if t.sons[0] == nil:
     LocalError(n.info, errXNeedsReturnType, "iterator")
+  # iterators are either 'inline' or 'closure':
+  if s.typ.callConv != ccInline: s.typ.callConv = ccClosure
   if n.sons[bodyPos].kind == nkEmpty and s.magic == mNone:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   
@@ -894,12 +896,13 @@ proc evalInclude(c: PContext, n: PNode): PNode =
   addSon(result, n)
   for i in countup(0, sonsLen(n) - 1): 
     var f = checkModuleName(n.sons[i])
-    var fileIndex = f.fileInfoIdx
+    if f.len > 0:
-    if ContainsOrIncl(c.includedFiles, fileIndex): 
+      var fileIndex = f.fileInfoIdx
-      LocalError(n.info, errRecursiveDependencyX, f.extractFilename)
+      if ContainsOrIncl(c.includedFiles, fileIndex): 
-    else:
+        LocalError(n.info, errRecursiveDependencyX, f.extractFilename)
-      addSon(result, semStmt(c, gIncludeFile(f)))
+      else:
-      Excl(c.includedFiles, fileIndex)
+        addSon(result, semStmt(c, gIncludeFile(f)))
+        Excl(c.includedFiles, fileIndex)
   
 proc setLine(n: PNode, info: TLineInfo) =
   for i in 0 .. <safeLen(n): setLine(n.sons[i], info)

compiler/semtypes.nim

@@ -872,7 +872,11 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
   of nkBlockType: result = semBlockType(c, n, prev)
   else:
     LocalError(n.info, errTypeExpected)
-    result = errorType(c)
+    if prev != nil and prev.kind == tyForward:
+      prev.kind = tyProxy
+      result = prev
+    else:
+      result = errorType(c)
   
 proc setMagicType(m: PSym, kind: TTypeKind, size: int) = 
   m.typ.kind = kind

compiler/sigmatch.nim

@@ -70,7 +70,7 @@ proc initCandidate*(c: var TCandidate, callee: PSym, binding: PNode,
   c.calleeSym = callee
   c.calleeScope = calleeScope
   initIdTable(c.bindings)
-  if binding != nil:
+  if binding != nil and callee.kind in RoutineKinds:
     var typeParams = callee.ast[genericParamsPos]
     for i in 1..min(sonsLen(typeParams), sonsLen(binding)-1):
       var formalTypeParam = typeParams.sons[i-1].typ

compiler/suggest.nim

@@ -214,7 +214,6 @@ proc suggestExpr*(c: PContext, node: PNode) =
     var n = findClosestDot(node)
     if n == nil: n = node
     else: cp = cpExact
-    
     if n.kind == nkDotExpr and cp == cpExact:
       var obj = safeSemExpr(c, n.sons[0])
       suggestFieldAccess(c, obj, outputs)