cursor and mutation tracking fixes (#15113)

* fixes #15110
* fixes #15096

* prepare varpartitions for cursor inference
* new cursor inference begins to work
* make tests green

compiler/ast.nim

@@ -1841,6 +1841,11 @@ proc isImportedException*(t: PType; conf: ConfigRef): bool =
 proc isInfixAs*(n: PNode): bool =
   return n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "as"
 
+proc skipColon*(n: PNode): PNode =
+  result = n
+  if n.kind == nkExprColonExpr:
+    result = n[1]
+
 proc findUnresolvedStatic*(n: PNode): PNode =
   # n.typ == nil: see issue #14802
   if n.kind == nkSym and n.typ != nil and n.typ.kind == tyStatic and n.typ.n == nil:
@@ -1860,6 +1865,10 @@ when false:
       if n[i].containsNil: return true
 
 template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
+
+proc hasDisabledAsgn*(t: PType): bool =
+  t.attachedOps[attachedAsgn] != nil and sfError in t.attachedOps[attachedAsgn].flags
+
 template incompleteType*(t: PType): bool =
   t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
 

compiler/cursor_inference.nim

@@ -1,300 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Cursor inference:
-## The basic idea was like this: Elide 'destroy(x)' calls if only
-## special literals are assigned to 'x' and 'x' is not mutated or
-## passed by 'var T' to something else. Special literals are string literals or
-## arrays / tuples of string literals etc.
-##
-## However, there is a much more general rule here: Compute which variables
-## can be annotated with `.cursor`. To see how and when we can do that,
-## think about this question: In `dest = src` when do we really have to
-## *materialize* the full copy? - Only if `dest` or `src` are mutated
-## afterwards. `dest` is the potential cursor variable, so that is
-## simple to analyse. And if `src` is a location derived from a
-## formal parameter, we also know it is not mutated! In other words, we
-## do a compile-time copy-on-write analysis.
-
-import
-  ast, types, renderer, idents, intsets, options
-
-type
-  Cursor = object
-    s: PSym
-    deps: IntSet
-  Con = object
-    cursors: seq[Cursor]
-    mayOwnData: IntSet
-    mutations: IntSet
-    reassigns: IntSet
-    config: ConfigRef
-    inAsgnSource: int
-
-proc locationRoot(e: PNode; followDotExpr = true): PSym =
-  var n = e
-  while true:
-    case n.kind
-    of nkSym:
-      if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar, skParam}:
-        return n.sym
-      else:
-        return nil
-    of nkDotExpr, nkDerefExpr, nkBracketExpr, nkHiddenDeref,
-        nkCheckedFieldExpr, nkAddr, nkHiddenAddr:
-      if followDotExpr:
-        n = n[0]
-      else:
-        return nil
-    of nkObjUpConv, nkObjDownConv:
-      n = n[0]
-    of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkCast:
-      n = n[1]
-    of nkStmtList, nkStmtListExpr:
-      if n.len > 0:
-        n = n[^1]
-      else:
-        return nil
-    else:
-      return nil
-
-proc addDep(c: var Con; dest: var Cursor; dependsOn: PSym) =
-  if dest.s != dependsOn:
-    dest.deps.incl dependsOn.id
-
-proc cursorId(c: Con; x: PSym): int =
-  for i in 0..<c.cursors.len:
-    if c.cursors[i].s == x: return i
-  return -1
-
-proc getCursors(c: Con) =
-  #[
-  Question: if x depends on y and y depends on z then also y depends on z.
-
-    Or does it?
-
-  var y = x # y gets the copy already
-
-  var harmless = x
-
-  y.s = "mutate"
-
-  ]#
-  for cur in c.cursors:
-    if not c.mayOwnData.contains(cur.s.id) and
-        cur.s.typ.skipTypes({tyGenericInst, tyAlias}).kind != tyOwned:
-      block doAdd:
-        for d in cur.deps:
-          # you cannot borrow from a location that is mutated or (owns data and is re-assigned)
-          if c.mutations.contains(d) or (c.mayOwnData.contains(d) and c.reassigns.contains(d)):
-            #echo "bah, not a cursor ", cur.s, " bad dependency ", d
-            break doAdd
-        when true:
-          cur.s.flags.incl sfCursor
-        when false:
-          echo "computed as a cursor ", cur.s, " ", cur.deps, " ", c.config $ cur.s.info
-
-proc analyseAsgn(c: var Con; dest: var Cursor; n: PNode) =
-  case n.kind
-  of nkEmpty, nkCharLit..pred(nkNilLit):
-    # primitive literals including the empty are harmless:
-    discard
-  of nkNilLit:
-    when false:
-      # XXX think about this case. Is 'nil' an owned location?
-      if hasDestructor(n.typ):
-        # 'myref = nil' is destructive and so 'myref' should not be a cursor:
-        c.mayOwnData.incl dest.s.id
-
-  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkCast, nkConv:
-    analyseAsgn(c, dest, n[1])
-
-  of nkIfStmt, nkIfExpr:
-    for i in 0..<n.len:
-      analyseAsgn(c, dest, n[i].lastSon)
-
-  of nkCaseStmt:
-    for i in 1..<n.len:
-      analyseAsgn(c, dest, n[i].lastSon)
-
-  of nkStmtList, nkStmtListExpr:
-    if n.len > 0:
-      analyseAsgn(c, dest, n[^1])
-
-  of nkClosure:
-    for i in 1..<n.len:
-      analyseAsgn(c, dest, n[i])
-    # you must destroy a closure:
-    c.mayOwnData.incl dest.s.id
-
-  of nkObjConstr:
-    for i in 1..<n.len:
-      analyseAsgn(c, dest, n[i])
-    if hasDestructor(n.typ):
-      # you must destroy a ref object:
-      c.mayOwnData.incl dest.s.id
-
-  of nkCurly, nkBracket, nkPar, nkTupleConstr:
-    for son in n:
-      analyseAsgn(c, dest, son)
-    if n.typ.skipTypes(abstractInst).kind == tySequence:
-      # you must destroy a sequence:
-      c.mayOwnData.incl dest.s.id
-
-  of nkSym:
-    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar, skParam}:
-      if n.sym.flags * {sfThread, sfGlobal} != {}:
-        # aliasing a global is inherently dangerous:
-        c.mayOwnData.incl dest.s.id
-      else:
-        # otherwise it's just a dependency, nothing to worry about:
-        c.addDep(dest, n.sym)
-
-  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
-      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr, nkAddr, nkHiddenAddr:
-    analyseAsgn(c, dest, n[0])
-
-  of nkCallKinds:
-    if hasDestructor(n.typ):
-      # calls do construct, what we construct must be destroyed,
-      # so dest cannot be a cursor:
-      c.mayOwnData.incl dest.s.id
-    elif n.typ.kind in {tyLent, tyVar}:
-      # we know the result is derived from the first argument:
-      let r = locationRoot(n[1])
-      if r != nil:
-        c.addDep(dest, r)
-    else:
-      let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone
-      # this list is subtle, we try to answer the question if after 'dest = f(src)'
-      # there is a connection betwen 'src' and 'dest' so that mutations to 'src'
-      # also reflect 'dest':
-      if magic in {mNone, mMove, mSlice, mAppendStrCh, mAppendStrStr, mAppendSeqElem, mArrToSeq}:
-        for i in 1..<n.len:
-          # we always have to assume a 'select(...)' like mechanism.
-          # But at least we do filter out simple POD types from the
-          # list of dependencies via the 'hasDestructor' check for
-          # the root's symbol.
-          if hasDestructor(n[i].typ.skipTypes({tyVar, tySink, tyLent, tyGenericInst, tyAlias})):
-            analyseAsgn(c, dest, n[i])
-
-  else:
-    # something we cannot handle:
-    c.mayOwnData.incl dest.s.id
-
-proc rhsIsSink(c: var Con, n: PNode) =
-  if n.kind == nkSym and n.typ.skipTypes(abstractInst-{tyOwned}).kind == tyRef:
-    discard "do no pessimize simple refs further, injectdestructors.nim will prevent moving from it"
-  else:
-    let r = locationRoot(n, followDotExpr = false)
-    if r != nil and r.kind != skParam:
-      # let x = cursor? --> treat it like a sink parameter
-      c.mayOwnData.incl r.id
-      c.mutations.incl r.id
-
-proc analyse(c: var Con; n: PNode) =
-  case n.kind
-  of nkCallKinds:
-    let parameters = n[0].typ
-    let L = if parameters != nil: parameters.len else: 0
-
-    analyse(c, n[0])
-    for i in 1..<n.len:
-      let it = n[i]
-      let r = locationRoot(it)
-      if r != nil and i < L:
-        let paramType = parameters[i].skipTypes({tyGenericInst, tyAlias})
-        if paramType.kind in {tyVar, tySink, tyOwned}:
-          # pass by var? mayOwnData the root
-          # Else we seek to take ownership of 'r'. This is only valid when 'r'
-          # actually owns its data. Thus 'r' cannot be a cursor:
-          c.mayOwnData.incl r.id
-          # and we assume it might get wasMoved(...) too:
-          c.mutations.incl r.id
-      analyse(c, it)
-
-  of nkAsgn, nkFastAsgn:
-    analyse(c, n[0])
-    inc c.inAsgnSource
-    analyse(c, n[1])
-    dec c.inAsgnSource
-
-    if n[0].kind == nkSym:
-      if hasDestructor(n[0].typ):
-        # re-assignment to the full object is fundamentally different:
-        let idx = cursorId(c, n[0].sym)
-        if idx >= 0:
-          analyseAsgn(c, c.cursors[idx], n[1])
-
-        c.reassigns.incl n[0].sym.id
-    else:
-      # assignments like 'x.field = value' mean that 'x' itself cannot
-      # be a cursor:
-      let r = locationRoot(n[0])
-      if r != nil:
-        # however, an assignment like 'it.field = x' does not influence r's
-        # cursorness property:
-        if r.typ.skipTypes(abstractInst).kind notin {tyPtr, tyRef}:
-          c.mayOwnData.incl r.id
-        c.mutations.incl r.id
-
-    if hasDestructor(n[1].typ):
-      rhsIsSink(c, n[1])
-
-  of nkAddr, nkHiddenAddr:
-    analyse(c, n[0])
-    let r = locationRoot(n[0])
-    if r != nil:
-      c.mayOwnData.incl r.id
-      c.mutations.incl r.id
-
-  of nkTupleConstr, nkBracket, nkObjConstr:
-    for child in n: analyse(c, child)
-    if c.inAsgnSource > 0:
-      for i in ord(n.kind == nkObjConstr)..<n.len:
-        if n[i].kind == nkSym:
-          # we assume constructions with cursors are better without
-          # the cursors because it's likely we can move then, see
-          # test arc/topt_no_cursor.nim
-          let r = n[i].sym
-          c.mayOwnData.incl r.id
-          c.mutations.incl r.id
-
-  of nkVarSection, nkLetSection:
-    for it in n:
-      let value = it[^1]
-      analyse(c, value)
-      if hasDestructor(it[0].typ):
-        for i in 0..<it.len-2:
-          let v = it[i]
-          if v.kind == nkSym and v.sym.flags * {sfThread, sfGlobal} == {}:
-            # assume it's a cursor:
-            c.cursors.add Cursor(s: it[i].sym)
-            if it.kind == nkVarTuple and value.kind in {nkPar, nkTupleConstr} and
-                it.len-2 == value.len:
-              # this might mayOwnData it again:
-              analyseAsgn(c, c.cursors[^1], value[i])
-              rhsIsSink(c, value[i])
-            else:
-              # this might mayOwnData it again:
-              analyseAsgn(c, c.cursors[^1], value)
-              rhsIsSink(c, value)
-
-  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef,
-      nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
-      nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
-      nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, nkTypeOfExpr:
-    discard
-  else:
-    for child in n: analyse(c, child)
-
-proc computeCursors*(n: PNode; config: ConfigRef) =
-  var c = Con(config: config)
-  analyse(c, n)
-  getCursors c

compiler/injectdestructors.nim

@@ -17,7 +17,7 @@ import
   intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents,
   strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
   lineinfos, parampatterns, sighashes, liftdestructors, optimizer,
-  cursor_inference
+  varpartitions
 
 from trees import exprStructuralEquivalent, getRoot
 
@@ -1002,7 +1002,7 @@ proc injectDestructorCalls*(g: ModuleGraph; owner: PSym; n: PNode): PNode =
     echoCfg(c.g)
     echo n
 
-  computeCursors(n, g.config)
+  computeCursors(owner, n, g.config)
 
   var scope: Scope
   let body = p(n, c, scope, normal)

compiler/sempass2.nim

@@ -826,6 +826,8 @@ proc trackCall(tracked: PEffects; n: PNode) =
       case op[i].kind
       of tySink:
         checkForSink(tracked.config, tracked.owner, n[i])
+      of tyVar:
+        tracked.hasDangerousAssign = true
       #of tyOut:
       # consider this case: p(out x, x); we want to remark that 'x' is not
       # initialized until after the call. Since we do this after we analysed the

compiler/varpartitions.nim

@@ -8,10 +8,10 @@
 #
 
 ## Partition variables into different graphs. Used for
-## Nim's write tracking. The used algorithm is "union find"
+## Nim's write tracking and also for the cursor inference.
-## with path compression.
+## The used algorithm is "union find" with path compression.
 
-import ast, types, lineinfos, options, msgs
+import ast, types, lineinfos, options, msgs, renderer
 from trees import getMagic
 from isolation_check import canAlias
 
@@ -20,16 +20,22 @@ type
     isMutated, # graph might be mutated
     connectsConstParam, # graph is connected to a non-var parameter.
 
+  VarFlag = enum
+    ownsData,
+    preventCursor
+
   VarIndexKind = enum
     isEmptyRoot,
     dependsOn,
     isRootOf
 
   VarIndex = object
+    flags: set[VarFlag]
     case kind: VarIndexKind
     of isEmptyRoot: discard
     of dependsOn: parent: int
     of isRootOf: graphIndex: int
+    sym: PSym
 
   MutationInfo* = object
     param: PSym
@@ -37,9 +43,10 @@ type
     flags: set[SubgraphFlag]
 
   Partitions = object
-    symToId: seq[PSym]
     s: seq[VarIndex]
     graphs: seq[MutationInfo]
+    unanalysableMutation, performCursorInference: bool
+    inAsgnSource: int
 
 proc `$`*(config: ConfigRef; g: MutationInfo): string =
   result = ""
@@ -56,8 +63,8 @@ proc `$`*(config: ConfigRef; g: MutationInfo): string =
       result.add config $ g.connectedVia
       result.add " is the statement that connected the mutation to the parameter"
 
-proc hasSideEffect(p: var Partitions; info: var MutationInfo): bool =
+proc hasSideEffect(c: var Partitions; info: var MutationInfo): bool =
-  for g in mitems p.graphs:
+  for g in mitems c.graphs:
     if g.flags == {isMutated, connectsConstParam}:
       info = g
       return true
@@ -65,17 +72,19 @@ proc hasSideEffect(p: var Partitions; info: var MutationInfo): bool =
 
 template isConstParam(a): bool = a.kind == skParam and a.typ.kind != tyVar
 
-proc registerVariable(p: var Partitions; n: PNode) =
+proc registerVariable(c: var Partitions; n: PNode) =
   if n.kind == nkSym:
-    p.symToId.add n.sym
     if isConstParam(n.sym):
-      p.s.add VarIndex(kind: isRootOf, graphIndex: p.graphs.len)
+      c.s.add VarIndex(kind: isRootOf, graphIndex: c.graphs.len, sym: n.sym)
-      p.graphs.add MutationInfo(param: n.sym, mutatedHere: unknownLineInfo,
+      c.graphs.add MutationInfo(param: n.sym, mutatedHere: unknownLineInfo,
                             connectedVia: unknownLineInfo, flags: {connectsConstParam})
     else:
-      p.s.add VarIndex(kind: isEmptyRoot)
+      c.s.add VarIndex(kind: isEmptyRoot, sym: n.sym)
 
-proc variableId(p: Partitions; x: PSym): int {.inline.} = system.find(p.symToId, x)
+proc variableId(c: Partitions; x: PSym): int {.inline.} =
+  for i in 0 ..< c.s.len:
+    if c.s[i].sym == x: return i
+  return -1
 
 proc root(v: var Partitions; start: int): int =
   result = start
@@ -88,7 +97,8 @@ proc root(v: var Partitions; start: int): int =
     var it = start
     while v.s[it].kind == dependsOn:
       let next = v.s[it].parent
-      v.s[it] = VarIndex(kind: dependsOn, parent: result)
+      v.s[it] = VarIndex(kind: dependsOn, parent: result,
+                         sym: v.s[it].sym, flags: v.s[it].flags)
       it = next
 
 proc potentialMutation(v: var Partitions; s: PSym; info: TLineInfo) =
@@ -97,7 +107,8 @@ proc potentialMutation(v: var Partitions; s: PSym; info: TLineInfo) =
     let r = root(v, id)
     case v.s[r].kind
     of isEmptyRoot:
-      v.s[r] = VarIndex(kind: isRootOf, graphIndex: v.graphs.len)
+      v.s[r] = VarIndex(kind: isRootOf, graphIndex: v.graphs.len,
+                        sym: v.s[r].sym, flags: v.s[r].flags)
       v.graphs.add MutationInfo(param: if isConstParam(s): s else: nil, mutatedHere: info,
                             connectedVia: unknownLineInfo, flags: {isMutated})
     of isRootOf:
@@ -110,7 +121,7 @@ proc potentialMutation(v: var Partitions; s: PSym; info: TLineInfo) =
     else:
       assert false, "cannot happen"
   else:
-    discard "we are not interested in the mutation"
+    v.unanalysableMutation = true
 
 proc connect(v: var Partitions; a, b: PSym; info: TLineInfo) =
   let aid = variableId(v, a)
@@ -142,10 +153,10 @@ proc connect(v: var Partitions; a, b: PSym; info: TLineInfo) =
       mutatedHere = gb.mutatedHere
       rbFlags = gb.flags
 
-    v.s[rb] = VarIndex(kind: dependsOn, parent: ra)
+    v.s[rb] = VarIndex(kind: dependsOn, parent: ra, sym: v.s[rb].sym, flags: v.s[rb].flags)
     case v.s[ra].kind
     of isEmptyRoot:
-      v.s[ra] = VarIndex(kind: isRootOf, graphIndex: v.graphs.len)
+      v.s[ra] = VarIndex(kind: isRootOf, graphIndex: v.graphs.len, sym: v.s[ra].sym, flags: v.s[ra].flags)
       v.graphs.add MutationInfo(param: param, mutatedHere: mutatedHere,
                             connectedVia: info, flags: paramFlags + rbFlags)
     of isRootOf:
@@ -157,32 +168,36 @@ proc connect(v: var Partitions; a, b: PSym; info: TLineInfo) =
     else:
       assert false, "cannot happen"
 
-proc allRoots(n: PNode; result: var seq[PSym]) =
+proc allRoots(n: PNode; result: var seq[PSym]; followDotExpr = true) =
   case n.kind
   of nkSym:
     if n.sym.kind in {skParam, skVar, skTemp, skLet, skResult, skForVar}:
       result.add(n.sym)
-  of nkHiddenDeref, nkDerefExpr, nkAddr, nkDotExpr, nkBracketExpr,
+
-      nkCheckedFieldExpr, nkHiddenAddr, nkObjUpConv, nkObjDownConv:
+  of nkDotExpr, nkDerefExpr, nkBracketExpr, nkHiddenDeref,
-    allRoots(n[0], result)
+      nkCheckedFieldExpr, nkAddr, nkHiddenAddr:
+    if followDotExpr:
+      allRoots(n[0], result, followDotExpr)
+
   of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkConv,
-      nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkCast:
+      nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkCast,
+      nkObjUpConv, nkObjDownConv:
     if n.len > 0:
-      allRoots(n.lastSon, result)
+      allRoots(n.lastSon, result, followDotExpr)
   of nkCaseStmt, nkObjConstr:
     for i in 1..<n.len:
-      allRoots(n[i].lastSon, result)
+      allRoots(n[i].lastSon, result, followDotExpr)
   of nkIfStmt, nkIfExpr:
     for i in 0..<n.len:
-      allRoots(n[i].lastSon, result)
+      allRoots(n[i].lastSon, result, followDotExpr)
   of nkBracket, nkTupleConstr, nkPar:
     for i in 0..<n.len:
-      allRoots(n[i], result)
+      allRoots(n[i], result, followDotExpr)
 
   of nkCallKinds:
     if n.typ != nil and n.typ.kind in {tyVar, tyLent}:
       if n.len > 1:
-        allRoots(n[1], result)
+        allRoots(n[1], result, followDotExpr)
     else:
       let m = getMagic(n)
       case m
@@ -201,77 +216,223 @@ proc allRoots(n: PNode; result: var seq[PSym]) =
             let paramType = typ.n[i].typ
             if not paramType.isCompileTimeOnly and not typ.sons[0].isEmptyType and
                 canAlias(paramType, typ.sons[0]):
-              allRoots(it, result)
+              allRoots(it, result, followDotExpr)
           else:
-            allRoots(it, result)
+            allRoots(it, result, followDotExpr)
 
       of mSlice:
-        allRoots(n[1], result)
+        allRoots(n[1], result, followDotExpr)
       else:
         discard "harmless operation"
   else:
     discard "nothing to do"
 
-proc deps(p: var Partitions; dest, src: PNode) =
+proc analyseAsgn(c: var Partitions; dest: var VarIndex; n: PNode) =
+  case n.kind
+  of nkEmpty, nkCharLit..nkNilLit:
+    # primitive literals including the empty are harmless:
+    discard
+
+  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkCast, nkConv:
+    analyseAsgn(c, dest, n[1])
+
+  of nkIfStmt, nkIfExpr:
+    for i in 0..<n.len:
+      analyseAsgn(c, dest, n[i].lastSon)
+
+  of nkCaseStmt:
+    for i in 1..<n.len:
+      analyseAsgn(c, dest, n[i].lastSon)
+
+  of nkStmtList, nkStmtListExpr:
+    if n.len > 0:
+      analyseAsgn(c, dest, n[^1])
+
+  of nkClosure:
+    for i in 1..<n.len:
+      analyseAsgn(c, dest, n[i])
+    # you must destroy a closure:
+    dest.flags.incl ownsData
+
+  of nkObjConstr:
+    for i in 1..<n.len:
+      analyseAsgn(c, dest, n[i])
+    if hasDestructor(n.typ):
+      # you must destroy a ref object:
+      dest.flags.incl ownsData
+
+  of nkCurly, nkBracket, nkPar, nkTupleConstr:
+    for son in n:
+      analyseAsgn(c, dest, son)
+    if n.typ.skipTypes(abstractInst).kind == tySequence:
+      # you must destroy a sequence:
+      dest.flags.incl ownsData
+
+  of nkSym:
+    if n.sym.kind in {skVar, skResult, skTemp, skLet, skForVar, skParam}:
+      if n.sym.flags * {sfThread, sfGlobal} != {}:
+        # aliasing a global is inherently dangerous:
+        dest.flags.incl ownsData
+      else:
+        # otherwise it's just a dependency, nothing to worry about:
+        connect(c, dest.sym, n.sym, n.info)
+
+  of nkDotExpr, nkBracketExpr, nkHiddenDeref, nkDerefExpr,
+      nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr, nkAddr, nkHiddenAddr:
+    analyseAsgn(c, dest, n[0])
+
+  of nkCallKinds:
+    if hasDestructor(n.typ):
+      # calls do construct, what we construct must be destroyed,
+      # so dest cannot be a cursor:
+      dest.flags.incl ownsData
+    elif n.typ.kind in {tyLent, tyVar}:
+      # we know the result is derived from the first argument:
+      var roots: seq[PSym]
+      allRoots(n[1], roots)
+      for r in roots:
+        connect(c, dest.sym, r, n[1].info)
+
+    else:
+      let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone
+      # this list is subtle, we try to answer the question if after 'dest = f(src)'
+      # there is a connection betwen 'src' and 'dest' so that mutations to 'src'
+      # also reflect 'dest':
+      if magic in {mNone, mMove, mSlice, mAppendStrCh, mAppendStrStr, mAppendSeqElem, mArrToSeq}:
+        for i in 1..<n.len:
+          # we always have to assume a 'select(...)' like mechanism.
+          # But at least we do filter out simple POD types from the
+          # list of dependencies via the 'hasDestructor' check for
+          # the root's symbol.
+          if hasDestructor(n[i].typ.skipTypes({tyVar, tySink, tyLent, tyGenericInst, tyAlias})):
+            analyseAsgn(c, dest, n[i])
+
+  else:
+    # something we cannot handle:
+    dest.flags.incl preventCursor
+
+proc noCursor(c: var Partitions, s: PSym) =
+  let vid = variableId(c, s)
+  if vid >= 0:
+    c.s[vid].flags.incl preventCursor
+
+proc rhsIsSink(c: var Partitions, n: PNode) =
+  if n.kind == nkSym and n.typ.skipTypes(abstractInst-{tyOwned}).kind == tyRef:
+    discard "do no pessimize simple refs further, injectdestructors.nim will prevent moving from it"
+  else:
+    var roots: seq[PSym]
+    allRoots(n, roots, followDotExpr = false)
+    # let x = cursor? --> treat it like a sink parameter
+    for r in roots:
+      noCursor(c, r)
+
+proc deps(c: var Partitions; dest, src: PNode) =
   var targets, sources: seq[PSym]
   allRoots(dest, targets)
   allRoots(src, sources)
   for t in targets:
     if dest.kind != nkSym:
-      potentialMutation(p, t, dest.info)
+      potentialMutation(c, t, dest.info)
 
     proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr}
     if types.searchTypeFor(t.typ, wrap):
       for s in sources:
-        connect(p, t, s, dest.info)
+        connect(c, t, s, dest.info)
 
-proc traverse(p: var Partitions; n: PNode) =
+  if c.performCursorInference and src.kind != nkEmpty:
+    if dest.kind == nkSym:
+      let vid = variableId(c, dest.sym)
+      if vid >= 0:
+        analyseAsgn(c, c.s[vid], src)
+        # do not borrow from a different local variable, this is easier
+        # than tracking reassignments, consider 'var cursor = local; local = newNode()'
+        if src.kind == nkSym and (src.sym.kind in {skVar, skResult, skTemp} or
+            (src.sym.kind in {skLet, skParam, skForVar} and hasDisabledAsgn(src.sym.typ))):
+          c.s[vid].flags.incl preventCursor
+
+    if hasDestructor(src.typ):
+      rhsIsSink(c, src)
+
+proc traverse(c: var Partitions; n: PNode) =
   case n.kind
   of nkLetSection, nkVarSection:
     for child in n:
       let last = lastSon(child)
-      traverse(p, last)
+      traverse(c, last)
       if child.kind == nkVarTuple and last.kind in {nkPar, nkTupleConstr}:
         if child.len-2 != last.len: return
         for i in 0..<child.len-2:
-          registerVariable(p, child[i])
+          registerVariable(c, child[i])
-          deps(p, child[i], last[i])
+          deps(c, child[i], last[i])
       else:
         for i in 0..<child.len-2:
-          registerVariable(p, child[i])
+          registerVariable(c, child[i])
-          deps(p, child[i], last)
+          deps(c, child[i], last)
   of nkAsgn, nkFastAsgn:
-    traverse(p, n[0])
+    traverse(c, n[0])
-    traverse(p, n[1])
+    inc c.inAsgnSource
-    deps(p, n[0], n[1])
+    traverse(c, n[1])
+    dec c.inAsgnSource
+    deps(c, n[0], n[1])
   of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef,
       nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
       nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
       nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, nkTypeOfExpr:
     discard "do not follow the construct"
   of nkCallKinds:
-    for child in n: traverse(p, child)
+    for child in n: traverse(c, child)
+
+    let parameters = n[0].typ
+    let L = if parameters != nil: parameters.len else: 0
 
-    if n[0].typ.isNil: return
-    var typ = skipTypes(n[0].typ, abstractInst)
-    if typ.kind != tyProc: return
-    assert(typ.len == typ.n.len)
     for i in 1..<n.len:
       let it = n[i]
-      if i < typ.len:
+      if i < L:
-        assert(typ.n[i].kind == nkSym)
+        let paramType = parameters[i].skipTypes({tyGenericInst, tyAlias})
-        let paramType = typ.n[i]
+        if not paramType.isCompileTimeOnly and paramType.kind in {tyVar, tySink, tyOwned}:
-        if paramType.typ.isCompileTimeOnly: continue
-        if paramType.typ.kind == tyVar:
           var roots: seq[PSym]
-          allRoots(n, roots)
+          allRoots(it, roots)
-          for r in roots: potentialMutation(p, r, n.info)
+          if paramType.kind == tyVar:
+            for r in roots: potentialMutation(c, r, it.info)
+          else:
+            for r in roots: noCursor(c, r)
+
+  of nkAddr, nkHiddenAddr:
+    traverse(c, n[0])
+    when false:
+      # XXX investigate if this is required, it doesn't look
+      # like it is!
+      var roots: seq[PSym]
+      allRoots(n[0], roots)
+      for r in roots:
+        potentialMutation(c, r, it.info)
+
+  of nkTupleConstr, nkBracket:
+    for child in n: traverse(c, child)
+    if c.inAsgnSource > 0:
+      for i in 0..<n.len:
+        if n[i].kind == nkSym:
+          # we assume constructions with cursors are better without
+          # the cursors because it's likely we can move then, see
+          # test arc/topt_no_cursor.nim
+          noCursor(c, n[i].sym)
+
+  of nkObjConstr:
+    for child in n: traverse(c, child)
+    if c.inAsgnSource > 0:
+      for i in 1..<n.len:
+        let it = n[i].skipColon
+        if it.kind == nkSym:
+          # we assume constructions with cursors are better without
+          # the cursors because it's likely we can move then, see
+          # test arc/topt_no_cursor.nim
+          noCursor(c, it.sym)
 
   else:
-    for child in n: traverse(p, child)
+    for child in n: traverse(c, child)
 
 proc mutatesNonVarParameters*(s: PSym; n: PNode; info: var MutationInfo): bool =
-  var par = Partitions()
+  var par = Partitions(performCursorInference: false)
   if s.kind != skMacro:
     let params = s.typ.n
     for i in 1..<params.len:
@@ -281,3 +442,25 @@ proc mutatesNonVarParameters*(s: PSym; n: PNode; info: var MutationInfo): bool =
 
   traverse(par, n)
   result = hasSideEffect(par, info)
+
+proc computeCursors*(s: PSym; n: PNode; config: ConfigRef) =
+  var par = Partitions(performCursorInference: true)
+  if s.kind notin {skMacro, skModule}:
+    let params = s.typ.n
+    for i in 1..<params.len:
+      registerVariable(par, params[i])
+    if resultPos < s.ast.safeLen:
+      registerVariable(par, s.ast[resultPos])
+
+  traverse(par, n)
+  for i in 0 ..< par.s.len:
+    let v = addr(par.s[i])
+    if v.flags == {} and v.sym.kind notin {skParam, skResult} and
+        v.sym.flags * {sfThread, sfGlobal} == {} and hasDestructor(v.sym.typ) and
+        v.sym.typ.skipTypes({tyGenericInst, tyAlias}).kind != tyOwned:
+      let rid = root(par, i)
+      if par.s[rid].kind == isRootOf and isMutated in par.graphs[par.s[rid].graphIndex].flags:
+        discard "cannot cursor into a graph that is mutated"
+      else:
+        v.sym.flags.incl sfCursor
+        #echo "this is now a cursor ", v.sym, " ", par.s[rid].flags

compiler/vmmarshal.nim

@@ -54,11 +54,6 @@ proc storeObj(s: var string; typ: PType; x: PNode; stored: var IntSet; conf: Con
       s.add(": ")
       storeAny(s, field.typ, it, stored, conf)
 
-proc skipColon*(n: PNode): PNode =
-  result = n
-  if n.kind == nkExprColonExpr:
-    result = n[1]
-
 proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet;
               conf: ConfigRef) =
   case t.kind

tests/arc/topt_cursor2.nim

@@ -0,0 +1,51 @@
+discard """
+  output: '''emptyemptyempty'''
+  cmd: '''nim c --gc:arc $file'''
+"""
+
+# bug #15039
+
+import lists
+
+type
+  Token = ref object of RootObj
+    children: DoublyLinkedList[Token]
+
+  Paragraph = ref object of Token
+
+method `$`(token: Token): string {.base.} =
+  result = "empty"
+
+method `$`(token: Paragraph): string =
+  if token.children.head == nil:
+    result = ""
+  else:
+    for c in token.children:
+      result.add $c
+
+proc parseLeafBlockInlines(token: Token) =
+  token.children.append(Token())
+  token.children.append(Token()) # <-- this one AAA
+
+  var emNode = newDoublyLinkedNode(Token())
+  var i = 0
+
+  var it = token.children.head
+  while it != nil:
+    var nxt = it.next  # this is not a cursor, it takes over ownership.
+    var childNode = it
+    if i == 0:
+      childNode.next = emNode # frees the object allocated in line 29 marked with AAA
+    elif i == 1:
+      emNode.next = childNode  #
+    it = nxt # incref on freed data, 'nxt' is freed
+    inc i
+
+proc parse() =
+  var token = Token()
+  token.children.append Paragraph()
+  parseLeafBlockInlines(token.children.head.value)
+  for children in token.children:
+    echo children
+
+parse()

tests/arc/topt_no_cursor.nim

@@ -29,11 +29,14 @@ result = (
 -- end of expandArc ------------------------
 --expandArc: delete
 
-var saved
+var
-var sibling_cursor = target.parent.left
+  sibling
-`=`(saved, sibling_cursor.right)
+  saved
-`=`(sibling_cursor.right, saved.left)
+`=`(sibling, target.parent.left)
-`=sink`(sibling_cursor.parent, saved)
+`=`(saved, sibling.right)
+`=`(sibling.right, saved.left)
+`=sink`(sibling.parent, saved)
+`=destroy`(sibling)
 -- end of expandArc ------------------------'''
 """