progress on deepCopy

compiler/ast.nim

@@ -263,7 +263,7 @@ type
     sfNamedParamCall, # symbol needs named parameter call syntax in target
                       # language; for interfacing with Objective C
     sfDiscardable,    # returned value may be discarded implicitly
-    sfDestructor,     # proc is destructor
+    sfOverriden,      # proc is overriden
     sfGenSym          # symbol is 'gensym'ed; do not add to symbol table
 
   TSymFlags* = set[TSymFlag]
@@ -785,12 +785,13 @@ type
                               # the body of the user-defined type class
                               # formal param list
                               # else: unused
-    destructor*: PSym         # destructor. warning: nil here may not necessary
-                              # mean that there is no destructor.
-                              # see instantiateDestructor in types.nim
     owner*: PSym              # the 'owner' of the type
     sym*: PSym                # types have the sym associated with them
                               # it is used for converting types to strings
+    destructor*: PSym         # destructor. warning: nil here may not necessary
+                              # mean that there is no destructor.
+                              # see instantiateDestructor in semdestruct.nim
+    deepCopy*: PSym           # overriden 'deepCopy' operation
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
     align*: int               # the type's alignment requirements
@@ -1190,6 +1191,7 @@ proc assignType(dest, src: PType) =
   dest.size = src.size
   dest.align = src.align
   dest.destructor = src.destructor
+  dest.deepCopy = src.deepCopy
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
     if dest.sym != nil:

compiler/ccgexprs.nim

@@ -359,6 +359,32 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   else: internalError("genAssignment: " & $ty.kind)
 
+proc genDeepCopy(p: BProc; dest, src: TLoc) =
+  var ty = skipTypes(dest.t, abstractRange)
+  case ty.kind
+  of tyPtr, tyRef, tyString, tyProc, tyTuple, tyObject, tyArray, tyArrayConstr:
+    # XXX optimize this
+    linefmt(p, cpsStmts, "#genericDeepCopy((void*)$1, (void*)$2, $3);$n",
+            addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
+  of tySequence:
+    linefmt(p, cpsStmts, "#genericSeqDeepCopy($1, $2, $3);$n",
+            addrLoc(dest), rdLoc(src), genTypeInfo(p.module, dest.t))
+  of tyOpenArray, tyVarargs:
+    linefmt(p, cpsStmts,
+         "#genericDeepCopyOpenArray((void*)$1, (void*)$2, $1Len0, $3);$n",
+         addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
+  of tySet:
+    if mapType(ty) == ctArray:
+      useStringh(p.module)
+      linefmt(p, cpsStmts, "memcpy((void*)$1, (NIM_CONST void*)$2, $3);$n",
+              rdLoc(dest), rdLoc(src), toRope(getSize(dest.t)))
+    else:
+      linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
+  of tyPointer, tyChar, tyBool, tyEnum, tyCString,
+     tyInt..tyUInt64, tyRange, tyVar:
+    linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
+  else: internalError("genDeepCopy: " & $ty.kind)
+
 proc getDestLoc(p: BProc, d: var TLoc, typ: PType) =
   if d.k == locNone: getTemp(p, typ, d)
 
@@ -1578,25 +1604,25 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mGetTypeInfo: genGetTypeInfo(p, e, d)
   of mSwap: genSwap(p, e, d)
   of mUnaryLt: 
-    if not (optOverflowCheck in p.options): unaryExpr(p, e, d, "$1 - 1")
+    if optOverflowCheck notin p.options: unaryExpr(p, e, d, "$1 - 1")
     else: unaryExpr(p, e, d, "#subInt($1, 1)")
   of mPred:
     # XXX: range checking?
-    if not (optOverflowCheck in p.options): binaryExpr(p, e, d, "$1 - $2")
+    if optOverflowCheck notin p.options: binaryExpr(p, e, d, "$1 - $2")
     else: binaryExpr(p, e, d, "#subInt($1, $2)")
   of mSucc:
     # XXX: range checking?
-    if not (optOverflowCheck in p.options): binaryExpr(p, e, d, "$1 + $2")
+    if optOverflowCheck notin p.options: binaryExpr(p, e, d, "$1 + $2")
     else: binaryExpr(p, e, d, "#addInt($1, $2)")
   of mInc:
-    if not (optOverflowCheck in p.options):
+    if optOverflowCheck notin p.options:
       binaryStmt(p, e, d, "$1 += $2;$n")
     elif skipTypes(e.sons[1].typ, abstractVar).kind == tyInt64:
       binaryStmt(p, e, d, "$1 = #addInt64($1, $2);$n")
     else:
       binaryStmt(p, e, d, "$1 = #addInt($1, $2);$n")
   of ast.mDec:
-    if not (optOverflowCheck in p.options):
+    if optOverflowCheck notin p.options:
       binaryStmt(p, e, d, "$1 -= $2;$n")
     elif skipTypes(e.sons[1].typ, abstractVar).kind == tyInt64:
       binaryStmt(p, e, d, "$1 = #subInt64($1, $2);$n")
@@ -1659,6 +1685,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mParallel:
     let n = semparallel.liftParallel(p.module.module, e)
     expr(p, n, d)
+  of mDeepCopy:
+    var a, b: TLoc
+    initLocExpr(p, e.sons[1], a)
+    initLocExpr(p, e.sons[2], b)
+    genDeepCopy(p, a, b) 
   else: internalError(e.info, "genMagicExpr: " & $op)
 
 proc genConstExpr(p: BProc, n: PNode): PRope
@@ -1878,7 +1909,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     of skVar, skForVar, skResult, skLet:
       if sfGlobal in sym.flags: genVarPrototype(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
-        internalError(n.info, "expr: var not init " & sym.name.s)
+        internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id
       if sfThread in sym.flags:
         accessThreadLocalVar(p, sym)
         if emulatedThreadVars(): 
@@ -1893,7 +1924,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       putLocIntoDest(p, d, sym.loc)
     of skParam:
       if sym.loc.r == nil or sym.loc.t == nil:
-        internalError(n.info, "expr: param not init " & sym.name.s)
+        #echo "FAILED FOR PRCO ", p.prc.name.s
+        internalError(n.info, "expr: param not init " & sym.name.s & "_" & $sym.id)
       putLocIntoDest(p, d, sym.loc)
     else: internalError(n.info, "expr(" & $sym.kind & "); unknown symbol")
   of nkNilLit:

compiler/ccgtypes.nim

@@ -453,7 +453,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: PRope,
     appf(result, " {$n", [name])
 
   var desc = getRecordFields(m, typ, check)
-  if (desc == nil) and not hasField: 
+  if desc == nil and not hasField: 
     appf(result, "char dummy;$n", [])
   else: 
     app(result, desc)
@@ -895,11 +895,20 @@ type
 
 include ccgtrav
 
+proc genDeepCopyProc(m: BModule; s: PSym; result: PRope) =
+  genProc(m, s)
+  appf(m.s[cfsTypeInit3], "$1.deepcopy = (N_NIMCALL_PTR(void*, void*)) $2;$n",
+     [result, s.loc.r])
+
 proc genTypeInfo(m: BModule, t: PType): PRope =
+  let origType = t
   var t = getUniqueType(t)
   result = ropef("NTI$1", [toRope(t.id)])
   if containsOrIncl(m.typeInfoMarker, t.id):
     return con("(&".toRope, result, ")".toRope)
+  
+  # getUniqueType doesn't skip tyDistinct when that has an overriden operation:
+  while t.kind == tyDistinct: t = t.lastSon
   let owner = t.skipTypes(typedescPtrs).owner.getModule
   if owner != m.module:
     # make sure the type info is created in the owner module
@@ -936,6 +945,10 @@ proc genTypeInfo(m: BModule, t: PType): PRope =
     # results are not deterministic!
     genTupleInfo(m, t, result)
   else: internalError("genTypeInfo(" & $t.kind & ')')
+  if t.deepCopy != nil:
+    genDeepCopyProc(m, t.deepCopy, result)
+  elif origType.deepCopy != nil:
+    genDeepCopyProc(m, origType.deepCopy, result)  
   result = con("(&".toRope, result, ")".toRope)
 
 proc genTypeSection(m: BModule, n: PNode) = 

compiler/ccgutils.nim

@@ -89,8 +89,10 @@ proc getUniqueType*(key: PType): PType =
   of tyTypeDesc, tyTypeClasses, tyGenericParam,
      tyFromExpr, tyFieldAccessor:
     internalError("GetUniqueType")
-  of tyGenericInst, tyDistinct, tyOrdinal, tyMutable,
+  of tyDistinct:
-     tyConst, tyIter, tyStatic:
+    if key.deepCopy != nil: result = key
+    else: result = getUniqueType(lastSon(key))
+  of tyGenericInst, tyOrdinal, tyMutable, tyConst, tyIter, tyStatic:
     result = getUniqueType(lastSon(key))
   of tyArrayConstr, tyGenericInvokation, tyGenericBody,
      tyOpenArray, tyArray, tySet, tyRange, tyTuple,

compiler/cgen.nim

@@ -295,6 +295,7 @@ proc postStmtActions(p: BProc) {.inline.} =
 
 proc accessThreadLocalVar(p: BProc, s: PSym)
 proc emulatedThreadVars(): bool {.inline.}
+proc genProc(m: BModule, prc: PSym)
 
 include "ccgtypes.nim"
 
@@ -574,7 +575,6 @@ proc fixLabel(p: BProc, labl: TLabel) =
 
 proc genVarPrototype(m: BModule, sym: PSym)
 proc requestConstImpl(p: BProc, sym: PSym)
-proc genProc(m: BModule, prc: PSym)
 proc genStmts(p: BProc, t: PNode)
 proc expr(p: BProc, n: PNode, d: var TLoc)
 proc genProcPrototype(m: BModule, sym: PSym)

compiler/lowerings.nim

@@ -122,8 +122,8 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode =
     if t == nil: break
     t = t.skipTypes(abstractInst)
   #if field == nil:
+  #  echo "FIELD ", b
   #  debug deref.typ
-  #  echo deref.typ.id
   internalAssert field != nil
   addSon(deref, a)
   result = newNodeI(nkDotExpr, info)
@@ -202,7 +202,8 @@ proc flowVarKind(t: PType): TFlowVarKind =
   elif containsGarbageCollectedRef(t): fvInvalid
   else: fvBlob
 
-proc addLocalVar(varSection: PNode; owner: PSym; typ: PType; v: PNode): PSym =
+proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType;
+                 v: PNode): PSym =
   result = newSym(skTemp, getIdent(genPrefix), owner, varSection.info)
   result.typ = typ
   incl(result.flags, sfFromGeneric)
@@ -210,8 +211,15 @@ proc addLocalVar(varSection: PNode; owner: PSym; typ: PType; v: PNode): PSym =
   var vpart = newNodeI(nkIdentDefs, varSection.info, 3)
   vpart.sons[0] = newSymNode(result)
   vpart.sons[1] = ast.emptyNode
-  vpart.sons[2] = v
+  vpart.sons[2] = if varInit.isNil: v else: ast.emptyNode
   varSection.add vpart
+  if varInit != nil:
+    let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
+    deepCopyCall.sons[0] = newSymNode(createMagic("deepCopy", mDeepCopy))
+    deepCopyCall.sons[1] = newSymNode(result)
+    deepCopyCall.sons[2] = v
+    deepCopyCall.typ = typ
+    varInit.add deepCopyCall
 
 discard """
 We generate roughly this:
@@ -244,24 +252,25 @@ stmtList:
 """
 
 proc createWrapperProc(f: PNode; threadParam, argsParam: PSym;
-                       varSection, call, barrier, fv: PNode;
+                       varSection, varInit, call, barrier, fv: PNode;
                        spawnKind: TSpawnResult): PSym =
   var body = newNodeI(nkStmtList, f.info)
   var threadLocalBarrier: PSym
   if barrier != nil:
     var varSection = newNodeI(nkVarSection, barrier.info)
-    threadLocalBarrier = addLocalVar(varSection, argsParam.owner, 
+    threadLocalBarrier = addLocalVar(varSection, nil, argsParam.owner, 
                                      barrier.typ, barrier)
     body.add varSection
     body.add callCodeGenProc("barrierEnter", threadLocalBarrier.newSymNode)
   var threadLocalProm: PSym
   if spawnKind == srByVar:
-    threadLocalProm = addLocalVar(varSection, argsParam.owner, fv.typ, fv)
+    threadLocalProm = addLocalVar(varSection, nil, argsParam.owner, fv.typ, fv)
   elif fv != nil:
     internalAssert fv.typ.kind == tyGenericInst
-    threadLocalProm = addLocalVar(varSection, argsParam.owner, fv.typ, fv)
+    threadLocalProm = addLocalVar(varSection, nil, argsParam.owner, fv.typ, fv)
-    
+  if barrier == nil:
     body.add varSection
+  body.add varInit
   if fv != nil and spawnKind != srByVar:
     # generate:
     #   fv.owner = threadParam
@@ -314,7 +323,8 @@ proc createCastExpr(argsParam: PSym; objType: PType): PNode =
   result.typ.rawAddSon(objType)
 
 proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym, 
-                             castExpr, call, varSection, result: PNode) =
+                             castExpr, call, 
+                             varSection, varInit, result: PNode) =
   let formals = n[0].typ.n
   let tmpName = getIdent(genPrefix)
   for i in 1 .. <n.len:
@@ -323,8 +333,8 @@ proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym,
     var argType = n[i].typ.skipTypes(abstractInst)
     if i < formals.len and formals[i].typ.kind == tyVar:
       localError(n[i].info, "'spawn'ed function cannot have a 'var' parameter")
-    elif containsTyRef(argType):
+    #elif containsTyRef(argType):
-      localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
+    #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
 
     let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
     var field = newSym(skField, fieldname, objType.owner, n.info)
@@ -332,7 +342,7 @@ proc setupArgsForConcurrency(n: PNode; objType: PType; scratchObj: PSym,
     objType.addField(field)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
 
-    let temp = addLocalVar(varSection, objType.owner, argType,
+    let temp = addLocalVar(varSection, varInit, objType.owner, argType,
                            indirectAccess(castExpr, field, n.info))
     call.add(newSymNode(temp))
 
@@ -367,7 +377,8 @@ proc genHigh(n: PNode): PNode =
     result.sons[1] = n
 
 proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
-                             castExpr, call, varSection, result: PNode) =
+                             castExpr, call, 
+                             varSection, varInit, result: PNode) =
   let formals = n[0].typ.n
   let tmpName = getIdent(genPrefix)
   # we need to copy the foreign scratch object fields into local variables
@@ -376,8 +387,8 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
     let n = n[i]
     let argType = skipTypes(if i < formals.len: formals[i].typ else: n.typ,
                             abstractInst)
-    if containsTyRef(argType):
+    #if containsTyRef(argType):
-      localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
+    #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
 
     let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
     var field = newSym(skField, fieldname, objType.owner, n.info)
@@ -403,7 +414,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
 
-        let threadLocal = addLocalVar(varSection, objType.owner, fieldA.typ,
+        let threadLocal = addLocalVar(varSection,nil, objType.owner, fieldA.typ,
                                       indirectAccess(castExpr, fieldA, n.info))
         slice.sons[2] = threadLocal.newSymNode
       else:
@@ -417,7 +428,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
       # the array itself does not need to go through a thread local variable:
       slice.sons[1] = genDeref(indirectAccess(castExpr, field, n.info))
 
-      let threadLocal = addLocalVar(varSection, objType.owner, fieldB.typ,
+      let threadLocal = addLocalVar(varSection,nil, objType.owner, fieldB.typ,
                                     indirectAccess(castExpr, fieldB, n.info))
       slice.sons[3] = threadLocal.newSymNode
       call.add slice
@@ -428,7 +439,7 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
       field.typ = a.typ
       objType.addField(field)
       result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
-      let threadLocal = addLocalVar(varSection, objType.owner, field.typ,
+      let threadLocal = addLocalVar(varSection,nil, objType.owner, field.typ,
                                     indirectAccess(castExpr, field, n.info))
       call.add(genDeref(threadLocal.newSymNode))
     else:
@@ -436,7 +447,8 @@ proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym;
       field.typ = argType
       objType.addField(field)
       result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
-      let threadLocal = addLocalVar(varSection, objType.owner, field.typ,
+      let threadLocal = addLocalVar(varSection, varInit,
+                                    objType.owner, field.typ,
                                     indirectAccess(castExpr, field, n.info))
       call.add(threadLocal.newSymNode)
 
@@ -504,10 +516,13 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
 
   call.add(fn)
   var varSection = newNodeI(nkVarSection, n.info)
+  var varInit = newNodeI(nkStmtList, n.info)
   if barrier.isNil:
-    setupArgsForConcurrency(n, objType, scratchObj, castExpr, call, varSection, result)
+    setupArgsForConcurrency(n, objType, scratchObj, castExpr, call, 
+                            varSection, varInit, result)
   else:
-    setupArgsForParallelism(n, objType, scratchObj, castExpr, call, varSection, result)
+    setupArgsForParallelism(n, objType, scratchObj, castExpr, call, 
+                            varSection, varInit, result)
 
   var barrierAsExpr: PNode = nil
   if barrier != nil:
@@ -539,7 +554,8 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
     fvAsExpr = indirectAccess(castExpr, field, n.info)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest))
 
-  let wrapper = createWrapperProc(fn, threadParam, argsParam, varSection, call,
+  let wrapper = createWrapperProc(fn, threadParam, argsParam, 
+                                  varSection, varInit, call,
                                   barrierAsExpr, fvAsExpr, spawnKind)
   result.add callCodeGenProc("nimSpawn", wrapper.newSymNode,
                              genAddrOf(scratchObj.newSymNode))

compiler/pragmas.nim

@@ -24,7 +24,8 @@ const
     wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, 
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
     wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
-    wGensym, wInject, wRaises, wTags, wUses, wOperator, wDelegator, wGcSafe}
+    wGensym, wInject, wRaises, wTags, wUses, wOperator, wDelegator, wGcSafe,
+    wOverride}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
@@ -628,10 +629,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           # implies nodecl, because otherwise header would not make sense
           if sym.loc.r == nil: sym.loc.r = toRope(sym.name.s)
         of wDestructor:
-          if sym.typ.sons.len == 2:
+          sym.flags.incl sfOverriden
-            sym.flags.incl sfDestructor
+          if sym.name.s.normalize != "destroy":
-          else:
+            localError(n.info, errGenerated, "destructor has to be named 'destroy'")
-            invalidPragma(it)
+        of wOverride:
+          sym.flags.incl sfOverriden
         of wNosideeffect:
           noVal(it)
           incl(sym.flags, sfNoSideEffect)

compiler/semdestruct.nim

@@ -125,8 +125,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType =
   # The destructor is either user-defined or automatically
   # generated by the compiler in a member-wise fashion.
   var t = skipTypes(typ, {tyConst, tyMutable}).skipGenericAlias
-  let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base
+  let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t
-                                  else: t
   
   if typeHoldingUserDefinition.destructor != nil:
     # XXX: This is not entirely correct for recursive types, but we need

compiler/semstmts.nim

@@ -1009,6 +1009,28 @@ proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
     addResult(c, s.typ.sons[0], n.info, s.kind)
     addResultNode(c, n)
 
+proc semOverride(c: PContext, s: PSym, n: PNode) =
+  case s.name.s.normalize
+  of "destroy": doDestructorStuff(c, s, n)
+  of "deepcopy":
+    if s.typ.len == 2 and
+        s.typ.sons[1].skipTypes(abstractInst).kind in {tyRef, tyPtr} and
+        sameType(s.typ.sons[1], s.typ.sons[0]):
+      # Note: we store the deepCopy in the base of the pointer to mitigate
+      # the problem that pointers are structural types:
+      let t = s.typ.sons[1].skipTypes(abstractInst).lastSon.skipTypes(abstractInst)
+      if t.kind in {tyObject, tyDistinct, tyEnum}:
+        t.deepCopy = s
+      else:
+        localError(n.info, errGenerated,
+                   "cannot bind 'deepCopy' to: " & typeToString(t))
+    else:
+      localError(n.info, errGenerated,
+                 "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
+  of "=": discard
+  else: localError(n.info, errGenerated,
+                   "'destroy' or 'deepCopy' expected for 'override'")
+
 type
   TProcCompilationSteps = enum
     stepRegisterSymbol,
@@ -1125,7 +1147,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     popOwner()
     pushOwner(s)
   s.options = gOptions
-  if sfDestructor in s.flags: doDestructorStuff(c, s, n)
+  if sfOverriden in s.flags: semOverride(c, s, n)
   if n.sons[bodyPos].kind != nkEmpty:
     # for DLL generation it is annoying to check for sfImportc!
     if sfBorrow in s.flags:

compiler/semtypinst.nim

@@ -137,7 +137,7 @@ proc hasGenericArguments*(n: PNode): bool =
     return false
 
 proc reResolveCallsWithTypedescParams(cl: var TReplTypeVars, n: PNode): PNode =
-  # This is needed fo tgenericshardcases
+  # This is needed for tgenericshardcases
   # It's possible that a generic param will be used in a proc call to a
   # typedesc accepting proc. After generic param substitution, such procs
   # should be optionally instantiated with the correct type. In order to

compiler/wordrecg.nim

@@ -39,7 +39,7 @@ type
 
     wDestroy,
     
-    wImmediate, wDestructor, wDelegator,
+    wImmediate, wDestructor, wDelegator, wOverride,
     wImportCpp, wImportObjC,
     wImportCompilerProc,
     wImportc, wExportc, wIncompleteStruct, wRequiresInit,
@@ -122,7 +122,7 @@ const
 
     "destroy",
     
-    "immediate", "destructor", "delegator",
+    "immediate", "destructor", "delegator", "override",
     "importcpp", "importobjc",
     "importcompilerproc", "importc", "exportc", "incompletestruct",
     "requiresinit", "align", "nodecl", "pure", "sideeffect",

doc/manual.txt

@@ -2497,7 +2497,7 @@ variable to use:
 
 When the compiler reaches the second ``add`` call, both ``a`` and ``b`` could
 be used with the proc, so one gets ``Error: expression '(a|b)' has no type (or
-is ambiguous)``. To solve this you would need to nest ``using`` with a
+is ambiguous)``. To solve this one would need to nest ``using`` with a
 ``block`` statement so as to control the reach of the ``using`` statement.
 
 If expression
@@ -4347,7 +4347,7 @@ Nimrod offers a special family of dot operators that can be used to
 intercept and rewrite proc call and field access attempts, referring
 to previously undeclared symbol names. They can be used to provide a
 fluent interface to objects lying outside the static confines of the
-Nimrod's type system such as values from dynamic scripting languages
+type system such as values from dynamic scripting languages
 or dynamic file formats such as JSON or XML.
 
 When Nimrod encounters an expression that cannot be resolved by the
@@ -4379,8 +4379,8 @@ This operator will be matched against both field accesses and method calls.
 operator `.()`
 ---------------
 This operator will be matched exclusively against method calls. It has higher
-precedence than the `.` operator and this allows you to handle expressions like
+precedence than the `.` operator and this allows one to handle expressions like
-`x.y` and `x.y()` differently if you are interfacing with a scripting language
+`x.y` and `x.y()` differently if one is interfacing with a scripting language
 for example.
 
 operator `.=`
@@ -4391,6 +4391,115 @@ This operator will be matched against assignments to missing fields.
   a.b = c # becomes `.=`(a, "b", c)
 
 
+Type bound operations
+=====================
+
+There are 3 operations that are bound to a type:
+
+1. Assignment
+2. Destruction
+3. Deep copying for communication between threads
+
+These operations can be *overriden* instead of *overloaded*. This means the
+implementation is automatically lifted to structured types. For instance if type
+``T`` has an overriden assignment operator ``=`` this operator is also used
+for assignments of the type ``seq[T]``. Since these operations are bound to a
+type they have to be bound to a nominal type for reasons of simplicity of
+implementation: This means an overriden ``deepCopy`` for ``ref T`` is really
+bound to ``T`` and not to ``ref T``. This also means that one cannot override
+``deepCopy`` for both ``ptr T`` and ``ref T`` at the same time; instead a
+helper distinct or object type has to be used for one pointer type.
+
+
+operator `=`
+------------
+
+This operator is the assignment operator. Note that in the contexts 
+like ``let v = expr``, ``var v = expr``, ``parameter = defaultValue`` or for
+parameter passing no assignment is performed. The ``override`` pragma is
+optional for overriding ``=``.
+
+
+destructors
+-----------
+
+A destructor must have a single parameter with a concrete type (the name of a
+generic type is allowed too). The name of the destructor has to be ``destroy``
+and it need to be annotated with the ``override`` pragma.
+
+``destroy(v)`` will be automatically invoked for every local stack 
+variable ``v`` that goes out of scope.
+
+If a structured type features a field with destructable type and 
+the user has not provided an explicit implementation, a destructor for the
+structured type will be automatically generated. Calls to any base class 
+destructors in both user-defined and generated destructors will be inserted.
+
+A destructor is attached to the type it destructs; expressions of this type
+can then only be used in *destructible contexts* and as parameters:
+
+.. code-block:: nimrod
+  type
+    TMyObj = object
+      x, y: int
+      p: pointer
+      
+  proc destroy(o: var TMyObj) {.override.} =
+    if o.p != nil: dealloc o.p
+    
+  proc open: TMyObj =
+    result = TMyObj(x: 1, y: 2, p: alloc(3))
+ 
+  proc work(o: TMyObj) =
+    echo o.x
+    # No destructor invoked here for 'o' as 'o' is a parameter.
+
+  proc main() =
+    # destructor automatically invoked at the end of the scope:
+    var x = open()
+    # valid: pass 'x' to some other proc:
+    work(x)
+    
+    # Error: usage of a type with a destructor in a non destructible context
+    echo open()
+
+A destructible context is currently only the following:
+
+1. The ``expr`` in ``var x = expr``.
+2. The ``expr`` in ``let x = expr``.
+3. The ``expr`` in ``return expr``.
+4. The ``expr`` in ``result = expr`` where ``result`` is the special symbol
+   introduced by the compiler.
+
+These rules ensure that the construction is tied to a variable and can easily
+be destructed at its scope exit. Later versions of the language will improve
+the support of destructors.
+
+Be aware that destructors are not called for objects allocated with ``new``.
+This may change in future versions of language, but for now the ``finalizer``
+parameter to ``new`` has to be used.
+
+
+deepCopy
+--------
+
+``deepCopy`` is a builtin that is invoked whenever data is passed to
+a ``spawn``'ed proc to ensure memory safety. The programmer can override its
+behaviour for a specific ``ref`` or ``ptr`` type ``T``. (Later versions of the
+language may weaken this restriction.)
+
+The signature has to be:
+
+.. code-block:: nimrod
+  proc deepCopy(x: T): T {.override.}
+
+This mechanism is used by most data structures that support shared memory like
+channels to implement thread safe automatic memory management.
+
+The builtin ``deepCopy`` can even clone closures and their environments. See
+the documentation of `spawn`_ for details.
+
+
 Term rewriting macros
 =====================
 
@@ -4985,62 +5094,14 @@ destructor pragma
 -----------------
 
 The ``destructor`` pragma is used to mark a proc to act as a type destructor.
-The proc must have a single parameter with a concrete type (the name of a
+Its usage is deprecated, use the ``override`` pragma instead.
-generic type is allowed too).
+See `type bound operations`_.
 
-Destructors will be automatically invoked when a local stack variable goes 
-out of scope.
 
-If a record type features a field with destructable type and 
+override pragma
-the user have not provided explicit implementation, Nimrod will automatically
+---------------
-generate a destructor for the record type. Nimrod will automatically insert
-calls to any base class destructors in both user-defined and generated
-destructors.
-
-A destructor is attached to the type it destructs; expressions of this type
-can then only be used in *destructible contexts* and as parameters:
-
-.. code-block:: nimrod
-  type
-    TMyObj = object
-      x, y: int
-      p: pointer
-      
-  proc destruct(o: var TMyObj) {.destructor.} =
-    if o.p != nil: dealloc o.p
-    
-  proc open: TMyObj =
-    result = TMyObj(x: 1, y: 2, p: alloc(3))
- 
-  proc work(o: TMyObj) =
-    echo o.x
-    # No destructor invoked here for 'o' as 'o' is a parameter.
-
-  proc main() =
-    # destructor automatically invoked at the end of the scope:
-    var x = open()
-    # valid: pass 'x' to some other proc:
-    work(x)
-    
-    # Error: usage of a type with a destructor in a non destructible context
-    echo open()
-
-A destructible context is currently only the following:
-
-1. The ``expr`` in ``var x = expr``.
-2. The ``expr`` in ``let x = expr``.
-3. The ``expr`` in ``return expr``.
-4. The ``expr`` in ``result = expr`` where ``result`` is the special symbol
-   introduced by the compiler.
-
-These rules ensure that the construction is tied to a variable and can easily
-be destructed at its scope exit. Later versions of the language will improve
-the support of destructors.
-
-Be aware that destructors are not called for objects allocated with ``new``.
-This may change in future versions of language, but for now use 
-the ``finalizer`` parameter to ``new``.
 
+See `type bound operations`_ instead.
 
 procvar pragma
 --------------
@@ -5490,10 +5551,10 @@ spelled*:
   proc printf(formatstr: cstring) {.header: "<stdio.h>", importc: "printf", varargs.}
 
 Note that this pragma is somewhat of a misnomer: Other backends will provide
-the same feature under the same name. Also, if you are interfacing with C++
+the same feature under the same name. Also, if one is interfacing with C++
-you can use the `ImportCpp pragma <nimrodc.html#importcpp-pragma>`_ and
+the `ImportCpp pragma <nimrodc.html#importcpp-pragma>`_ and
 interfacing with Objective-C the `ImportObjC pragma
-<nimrodc.html#importobjc-pragma>`_.
+<nimrodc.html#importobjc-pragma>`_ can be used.
 
 
 Exportc pragma
@@ -5781,12 +5842,14 @@ used instead. `spawn`:idx: is used to pass a task to the thread pool:
 
 Currently the expression that ``spawn`` takes is however quite restricted: 
 
-* It must be a call expresion ``f(a, ...)``.
+* It must be a call expression ``f(a, ...)``.
 * ``f`` must be ``gcsafe``.
 * ``f`` must not have the calling convention ``closure``.
-* ``f``'s parameters may not be of type ``var`` nor may they contain ``ref``.
+* ``f``'s parameters may not be of type ``var``.
-  This means you have to use raw ``ptr``'s for data passing reminding the
+  This means one has to use raw ``ptr``'s for data passing reminding the
   programmer to be careful.
+* ``ref`` parameters are deeply copied which is a subtle semantic change and
+  can cause performance problems but ensures memory safety.
 * For *safe* data exchange between ``f`` and the caller a global ``TChannel``
   needs to be used. Other means will be provided soon.
 

lib/system.nim

@@ -2997,11 +2997,14 @@ proc locals*(): TObject {.magic: "Locals", noSideEffect.} =
   ##   # -> B is 1
   discard
 
-proc deepCopy*[T](x: T): T {.magic: "DeepCopy", noSideEffect.} =
+when hostOS != "standalone" and not defined(NimrodVM) and not defined(JS):
+  proc deepCopy*[T](x: var T, y: T) {.noSideEffect, magic: "DeepCopy".} =
     ## performs a deep copy of `x`. This is also used by the code generator
     ## for the implementation of ``spawn``.
     discard
 
+  include "system/deepcopy"
+
 {.pop.} #{.push warning[GcMem]: off.}
 
 when not defined(booting):

lib/system/deepcopy.nim

@@ -0,0 +1,121 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) {.gcsafe.}
+proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.gcsafe.} =
+  var
+    d = cast[TAddress](dest)
+    s = cast[TAddress](src)
+  case n.kind
+  of nkSlot:
+    genericDeepCopyAux(cast[pointer](d +% n.offset), 
+                       cast[pointer](s +% n.offset), n.typ)
+  of nkList:
+    for i in 0..n.len-1:
+      genericDeepCopyAux(dest, src, n.sons[i])
+  of nkCase:
+    var dd = selectBranch(dest, n)
+    var m = selectBranch(src, n)
+    # reset if different branches are in use; note different branches also
+    # imply that's not self-assignment (``x = x``)!
+    if m != dd and dd != nil: 
+      genericResetAux(dest, dd)
+    copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset),
+            n.typ.size)
+    if m != nil:
+      genericDeepCopyAux(dest, src, m)
+  of nkNone: sysAssert(false, "genericDeepCopyAux")
+
+proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
+  var
+    d = cast[TAddress](dest)
+    s = cast[TAddress](src)
+  sysAssert(mt != nil, "genericDeepCopyAux 2")
+  case mt.kind
+  of tyString:
+    var x = cast[PPointer](dest)
+    var s2 = cast[PPointer](s)[]
+    if s2 == nil:
+      unsureAsgnRef(x, s2)
+    else:
+      unsureAsgnRef(x, copyString(cast[NimString](s2)))
+  of tySequence:
+    var s2 = cast[PPointer](src)[]
+    var seq = cast[PGenericSeq](s2)      
+    var x = cast[PPointer](dest)
+    if s2 == nil:
+      unsureAsgnRef(x, s2)
+      return
+    sysAssert(dest != nil, "genericDeepCopyAux 3")
+    unsureAsgnRef(x, newSeq(mt, seq.len))
+    var dst = cast[TAddress](cast[PPointer](dest)[])
+    for i in 0..seq.len-1:
+      genericDeepCopyAux(
+        cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
+        cast[pointer](cast[TAddress](s2) +% i *% mt.base.size +%
+                     GenericSeqSize),
+        mt.base)
+  of tyObject:
+    # we need to copy m_type field for tyObject, as it could be empty for
+    # sequence reallocations:
+    var pint = cast[ptr PNimType](dest)
+    pint[] = cast[ptr PNimType](src)[]
+    if mt.base != nil:
+      genericDeepCopyAux(dest, src, mt.base)
+    genericDeepCopyAux(dest, src, mt.node)
+  of tyTuple:
+    genericDeepCopyAux(dest, src, mt.node)
+  of tyArray, tyArrayConstr:
+    for i in 0..(mt.size div mt.base.size)-1:
+      genericDeepCopyAux(cast[pointer](d +% i*% mt.base.size),
+                         cast[pointer](s +% i*% mt.base.size), mt.base)
+  of tyRef:
+    var z: pointer
+    if mt.base.deepCopy != nil:
+      z = mt.base.deepCopy(cast[PPointer](s)[])
+    else:
+      # we modify the header of the cell temporarily; instead of the type
+      # field we store a forwarding pointer. XXX This is bad when the cloning
+      # fails due to OOM etc.
+      let x = usrToCell(cast[PPointer](s)[])
+      let forw = cast[int](x.typ)
+      if (forw and 1) == 1:
+        # we stored a forwarding pointer, so let's use that:
+        z = cast[pointer](forw and not 1)
+      else:
+        let realType = x.typ
+        z = newObj(realType, realType.base.size)
+        x.typ = cast[PNimType](cast[int](z) or 1)
+        genericDeepCopyAux(dest, addr(z), realType)
+        x.typ = realType
+    unsureAsgnRef(cast[PPointer](dest), z)
+  of tyPtr:
+    # no cycle check here, but also not really required
+    if mt.base.deepCopy != nil:
+      cast[PPointer](dest)[] = mt.base.deepCopy(cast[PPointer](s)[])
+    else:
+      cast[PPointer](dest)[] = cast[PPointer](s)[]
+  else:
+    copyMem(dest, src, mt.size) # copy raw bits
+
+proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =
+  genericDeepCopyAux(dest, src, mt)
+
+proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =
+  var src = src
+  genericDeepCopy(dest, addr(src), mt)
+
+proc genericDeepCopyOpenArray(dest, src: pointer, len: int,
+                            mt: PNimType) {.compilerproc.} =
+  var
+    d = cast[TAddress](dest)
+    s = cast[TAddress](src)
+  for i in 0..len-1:
+    genericDeepCopy(cast[pointer](d +% i*% mt.base.size),
+                    cast[pointer](s +% i*% mt.base.size), mt.base)

lib/system/hti.nim

@@ -86,6 +86,7 @@ type
     node: ptr TNimNode # valid for tyRecord, tyObject, tyTuple, tyEnum
     finalizer: pointer # the finalizer for the type
     marker: proc (p: pointer, op: int) {.nimcall, gcsafe.} # marker proc for GC
+    deepcopy: proc (p: pointer): pointer {.nimcall, gcsafe.}
   PNimType = ptr TNimType
   
 # node.len may be the ``first`` element of a set

tests/destructor/tdestructor.nim

@@ -55,14 +55,14 @@ type
         q: TMyGeneric3[TMyObj, int, int]
       r: string
 
-proc destruct(o: var TMyObj) {.destructor.} =
+proc destroy(o: var TMyObj) {.override.} =
   if o.p != nil: dealloc o.p
   echo "myobj destroyed"
 
-proc destroy(o: var TMyGeneric1) {.destructor.} =
+proc destroy(o: var TMyGeneric1) {.override.} =
   echo "mygeneric1 destroyed"
 
-proc destroy[A, B](o: var TMyGeneric2[A, B]) {.destructor.} =
+proc destroy[A, B](o: var TMyGeneric2[A, B]) {.override.} =
   echo "mygeneric2 destroyed"
 
 proc open: TMyObj =

tests/destructor/tdestructor2.nim

@@ -8,7 +8,7 @@ type
     x, y: int
     p: pointer
     
-proc destruct(o: var TMyObj) {.destructor.} =
+proc destroy(o: var TMyObj) {.override.} =
   if o.p != nil: dealloc o.p
   
 proc open: TMyObj =

todo.txt

@@ -3,20 +3,26 @@ version 0.9.6
 
 - scopes are still broken for generic instantiation!
 - start experimental branch
-- overloading of '='; general lift mechanism
+- implicit deref for parameter matching
 
 Concurrency
 -----------
 
 - 'gcsafe' inferrence needs to be fixed
+- 'deepCopy' needs to be instantiated for
+  generics *when the type is constructed*
+- test 'deepCopy'
+- overloading of '='; general lift mechanism
+
 - the disjoint checker needs to deal with 'a = spawn f(); g = spawn f()'
-- implement 'deepCopy' builtin
 - implement 'foo[1..4] = spawn(f[4..7])'
-- support for exception propagation
+- document the new 'spawn' and 'parallel' statements
-- Minor: The copying of the 'ref Promise' into the thead local storage only
+
+Low priority:
+- support for exception propagation? (hard to implement)
+- the copying of the 'ref Promise' into the thead local storage only
   happens to work due to the write barrier's implementation
 - implement lock levels --> first without the more complex race avoidance
-- document the new 'spawn' and 'parallel' statements
 
 
 Misc
@@ -27,8 +33,6 @@ Misc
 - fix the tuple unpacking in lambda bug
 - make tuple unpacking work in a non-var/let context
 - special rule for ``[]=``
-- ``=`` should be overloadable; requires specialization for ``=``; general
-  lift mechanism in the compiler is already implemented for 'fields'
 - built-in 'getImpl'
 - type API for macros; make 'spawn' a macro
 - markAndSweepGC should expose an API for fibers
@@ -57,13 +61,9 @@ version 0.9.x
 - implement 'bits' pragmas
 - we need a magic thisModule symbol
 - provide --nilChecks:on|off
-- fix closures/lambdalifting
 - ensure (ref T)(a, b) works as a type conversion and type constructor
 - optimize 'genericReset'; 'newException' leads to code bloat
 - stack-less GC
-- implicit deref for parameter matching
-
-- VM: optimize opcAsgnStr
 
 
 version 0.9.X

web/news.txt

@@ -20,6 +20,14 @@ News
     representation.
   - ``uri.TUrl`` as well as the ``parseurl`` module are now deprecated in favour
     of the new ``TUri`` type in the ``uri`` module.
+  - The ``destructor`` pragma has been deprecated. Use the ``override`` pragma
+    instead. The destructor's name has to be ``destroy`` now.
+
+  Language Additions
+  ------------------
+
+  - There is a new ``parallel`` statement for safe fork&join parallel computing.
+  
 
   Library Additions
   -----------------