.cursor implementation (#12637)

* cursors: first implementation
* added currently failing test
* .cursor works for doubly linked lists
* make -d:useMalloc work again
* added code to nil out refs in a destructor
* it's now called --gc:arc
* renderer.nim: render nkBreakState properly
* make simple closure iterators work without leaking

compiler/ast.nim

@@ -228,7 +228,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # already 38 flags!
+  TSymFlag* = enum    # 39 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -284,6 +284,7 @@ type
                       # variable is generated closure environment; requires early
                       # destruction for --newruntime.
     sfTemplateParam   # symbol is a template parameter
+    sfCursor          # variable/field is a cursor, see RFC 177 for details
 
 
   TSymFlags* = set[TSymFlag]

compiler/commands.nim

@@ -224,15 +224,15 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   case switch.normalize
   of "gc":
     case arg.normalize
-    of "boehm":        result = conf.selectedGC == gcBoehm
+    of "boehm": result = conf.selectedGC == gcBoehm
-    of "refc":         result = conf.selectedGC == gcRefc
+    of "refc": result = conf.selectedGC == gcRefc
-    of "v2":           result = false
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
-    of "destructors":  result = conf.selectedGC == gcDestructors
+    of "destructors", "arc": result = conf.selectedGC == gcDestructors
-    of "hooks":        result = conf.selectedGC == gcHooks
+    of "hooks": result = conf.selectedGC == gcHooks
-    of "go":           result = conf.selectedGC == gcGo
+    of "go": result = conf.selectedGC == gcGo
-    of "none":         result = conf.selectedGC == gcNone
+    of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
@@ -244,9 +244,9 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   of "verbosity": result = $conf.verbosity == arg
   of "app":
     case arg.normalize
-    of "gui":       result = contains(conf.globalOptions, optGenGuiApp)
+    of "gui": result = contains(conf.globalOptions, optGenGuiApp)
-    of "console":   result = not contains(conf.globalOptions, optGenGuiApp)
+    of "console": result = not contains(conf.globalOptions, optGenGuiApp)
-    of "lib":       result = contains(conf.globalOptions, optGenDynLib) and
+    of "lib": result = contains(conf.globalOptions, optGenDynLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
@@ -453,7 +453,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "markandsweep":
         conf.selectedGC = gcMarkAndSweep
         defineSymbol(conf.symbols, "gcmarkandsweep")
-      of "destructors":
+      of "destructors", "arc":
         conf.selectedGC = gcDestructors
         defineSymbol(conf.symbols, "gcdestructors")
         incl conf.globalOptions, optSeqDestructors

compiler/condsyms.nim

@@ -101,3 +101,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimFixedForwardGeneric")
   defineSymbol("nimnomagic64")
   defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasCursor")

compiler/injectdestructors.nim

@@ -418,6 +418,17 @@ proc pArg(arg: PNode; c: var Con; isSink: bool): PNode =
   else:
     result = p(arg, c)
 
+proc isCursor(n: PNode): bool =
+  case n.kind
+  of nkSym:
+    result = sfCursor in n.sym.flags
+  of nkDotExpr:
+    result = sfCursor in n[1].sym.flags
+  of nkCheckedFieldExpr:
+    result = isCursor(n[0])
+  else:
+    result = false
+
 proc p(n: PNode; c: var Con): PNode =
   case n.kind
   of nkCallKinds:
@@ -459,7 +470,7 @@ proc p(n: PNode; c: var Con): PNode =
       if it.kind == nkVarTuple and hasDestructor(ri.typ):
         let x = lowerTupleUnpacking(c.graph, it, c.owner)
         result.add p(x, c)
-      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ):
+      elif it.kind == nkIdentDefs and hasDestructor(it[0].typ) and not isCursor(it[0]):
         for j in 0..<it.len-2:
           let v = it[j]
           if v.kind == nkSym:
@@ -483,7 +494,8 @@ proc p(n: PNode; c: var Con): PNode =
         v.add itCopy
         result.add v
   of nkAsgn, nkFastAsgn:
-    if hasDestructor(n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda}:
+    if hasDestructor(n[0].typ) and n[1].kind notin {nkProcDef, nkDo, nkLambda} and
+        not isCursor(n[0]):
       # rule (self-assignment-removal):
       if n[1].kind == nkSym and n[0].kind == nkSym and n[0].sym == n[1].sym:
         result = newNodeI(nkEmpty, n.info)
@@ -515,7 +527,7 @@ proc p(n: PNode; c: var Con): PNode =
   of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef,
       nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef,
       nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, nkExportStmt,
-      nkPragma, nkCommentStmt, nkBreakStmt:
+      nkPragma, nkCommentStmt, nkBreakStmt, nkBreakState:
     result = n
   of nkWhileStmt:
     result = copyNode(n)

compiler/liftdestructors.nim

@@ -52,11 +52,24 @@ proc dotField(x: PNode, f: PSym): PNode =
   result.sons[1] = newSymNode(f, x.info)
   result.typ = f.typ
 
+proc newAsgnStmt(le, ri: PNode): PNode =
+  result = newNodeI(nkAsgn, le.info, 2)
+  result.sons[0] = le
+  result.sons[1] = ri
+
+proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  if c.kind != attachedDestructor:
+    body.add newAsgnStmt(x, y)
+
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode) =
   case n.kind
   of nkSym:
     let f = n.sym
-    fillBody(c, f.typ, body, x.dotField(f), y.dotField(f))
+    if sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
+        c.g.config.selectedGC == gcDestructors:
+      defaultOp(c, f.typ, body, x.dotField(f), y.dotField(f))
+    else:
+      fillBody(c, f.typ, body, x.dotField(f), y.dotField(f))
   of nkNilLit: discard
   of nkRecCase:
     if c.kind in {attachedSink, attachedAsgn, attachedDeepCopy}:
@@ -113,11 +126,6 @@ proc newAsgnCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode =
   result.add genAddr(g, x)
   result.add y
 
-proc newAsgnStmt(le, ri: PNode): PNode =
-  result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
-
 proc newOpCall(op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ.sons[0])
   result.add(newSymNode(op))
@@ -236,10 +244,6 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add newDeepCopyCall(op, x, y)
       result = true
 
-proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
-  if c.kind != attachedDestructor:
-    body.add newAsgnStmt(x, y)
-
 proc addVar(father, v, value: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
   vpart.sons[0] = v
@@ -393,6 +397,9 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
+    when false:
+      # XXX investigate if this is necessary:
+      actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
 
@@ -419,6 +426,9 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
+    when false:
+      # XXX investigate if this is necessary:
+      actions.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
 

compiler/pragmas.nim

@@ -62,11 +62,11 @@ const
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
     wBorrow, wGcSafe, wPartial, wExplain, wPackage}
   fieldPragmas* = declPragmas + {
-    wGuard, wBitsize} - {wExportNims, wNodecl} # why exclude these?
+    wGuard, wBitsize, wCursor} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
     wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor}
   constPragmas* = declPragmas + {wHeader, wMagic,
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore}
@@ -833,6 +833,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wVolatile:
         noVal(c, it)
         incl(sym.flags, sfVolatile)
+      of wCursor:
+        noVal(c, it)
+        incl(sym.flags, sfCursor)
       of wRegister:
         noVal(c, it)
         incl(sym.flags, sfRegister)

compiler/renderer.nim

@@ -1526,6 +1526,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
 
   of nkBreakState:
     put(g, tkTuple, "breakstate")
+    if renderIds in g.flags:
+      gsons(g, n, c, 0)
   of nkTypeClassTy:
     gTypeClassTy(g, n)
   else:

compiler/wordrecg.nim

@@ -35,7 +35,7 @@ type
     wColon, wColonColon, wEquals, wDot, wDotDot,
     wStar, wMinus,
     wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks,
-    wIntDefine, wStrDefine, wBoolDefine
+    wIntDefine, wStrDefine, wBoolDefine, wCursor,
 
     wImmediate, wConstructor, wDestructor, wDelegator, wOverride,
     wImportCpp, wImportObjC,
@@ -121,7 +121,7 @@ const
     ":", "::", "=", ".", "..",
     "*", "-",
     "magic", "thread", "final", "profiler", "memtracker", "objchecks",
-    "intdefine", "strdefine", "booldefine",
+    "intdefine", "strdefine", "booldefine", "cursor",
 
     "immediate", "constructor", "destructor", "delegator", "override",
     "importcpp", "importobjc",

lib/core/allocators.nim

@@ -43,7 +43,7 @@ proc getLocalAllocator*(): Allocator =
     result = addr allocatorStorage
     result.alloc = proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall, raises: [].} =
       when defined(useMalloc) and not defined(nimscript):
-        result = c_malloc(size)
+        result = c_malloc(cuint size)
         # XXX do we need this?
         nimZeroMem(result, size)
       else:
@@ -57,7 +57,7 @@ proc getLocalAllocator*(): Allocator =
       inc a.deallocCount
     result.realloc = proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall, raises: [].} =
       when defined(useMalloc) and not defined(nimscript):
-        result = c_realloc(p, newSize)
+        result = c_realloc(p, cuint newSize)
       else:
         result = system.realloc(p, newSize)
       nimZeroMem(result +! oldSize, newSize - oldSize)

lib/core/runtime_v2.nim

@@ -47,7 +47,7 @@ proc nimNewObj(size: int): pointer {.compilerRtl.} =
   when defined(nimscript):
     discard
   elif defined(useMalloc):
-    var orig = c_malloc(s)
+    var orig = c_malloc(cuint s)
     nimZeroMem(orig, s)
     result = orig +! sizeof(RefHeader)
   else:

lib/pure/collections/lists.nim

@@ -76,13 +76,16 @@
 when not defined(nimhygiene):
   {.pragma: dirty.}
 
+when not defined(nimHasCursor):
+  {.pragma: cursor.}
+
 type
   DoublyLinkedNodeObj*[T] = object ## \
     ## A node a doubly linked list consists of.
     ##
     ## It consists of a `value` field, and pointers to `next` and `prev`.
     next*: <//>(ref DoublyLinkedNodeObj[T])
-    prev*: ref DoublyLinkedNodeObj[T]
+    prev* {.cursor.}: ref DoublyLinkedNodeObj[T]
     value*: T
   DoublyLinkedNode*[T] = ref DoublyLinkedNodeObj[T]
 
@@ -100,7 +103,7 @@ type
     ## Use `initSinglyLinkedList proc <#initSinglyLinkedList>`_ to create
     ## a new empty list.
     head*: <//>(SinglyLinkedNode[T])
-    tail*: SinglyLinkedNode[T]
+    tail* {.cursor.}: SinglyLinkedNode[T]
 
   DoublyLinkedList*[T] = object ## \
     ## A doubly linked list.
@@ -108,7 +111,7 @@ type
     ## Use `initDoublyLinkedList proc <#initDoublyLinkedList>`_ to create
     ## a new empty list.
     head*: <//>(DoublyLinkedNode[T])
-    tail*: DoublyLinkedNode[T]
+    tail* {.cursor.}: DoublyLinkedNode[T]
 
   SinglyLinkedRing*[T] = object ## \
     ## A singly linked ring.
@@ -116,7 +119,7 @@ type
     ## Use `initSinglyLinkedRing proc <#initSinglyLinkedRing>`_ to create
     ## a new empty ring.
     head*: <//>(SinglyLinkedNode[T])
-    tail*: SinglyLinkedNode[T]
+    tail* {.cursor.}: SinglyLinkedNode[T]
 
   DoublyLinkedRing*[T] = object ## \
     ## A doubly linked ring.

tests/destructor/tlists.nim

@@ -0,0 +1,38 @@
+discard """
+  output: '''Success
+@["a", "b", "c"]
+0'''
+  cmd: '''nim c --gc:destructors $file'''
+"""
+
+import os
+import math
+import lists
+import strutils
+
+proc mkleak() =
+  # allocate 1 MB via linked lists
+  let numberOfLists = 100
+  for i in countUp(1, numberOfLists):
+    var leakList = initDoublyLinkedList[string]()
+    let numberOfLeaks = 5000
+    for j in countUp(1, numberOfLeaks):
+      leakList.append(newString(200))
+
+proc mkManyLeaks() =
+  for i in 0..0:
+    mkleak()
+  echo "Success"
+
+iterator foobar(c: string): seq[string] {.closure.} =
+  yield @["a", "b", c]
+
+proc tsimpleClosureIterator =
+  var myc = "c"
+  for it in foobar(myc):
+    echo it
+
+let startMem = getOccupiedMem()
+mkManyLeaks()
+tsimpleClosureIterator()
+echo getOccupiedMem() - startMem