newruntime for async (#11650)

* fixes overloading resolution for passing owned(Future[string]) to Future[T]
* WIP: make --newruntime work with .async
* memtracker: make it compile again
* make Nimble compile again

compiler/condsyms.nim

@@ -94,3 +94,5 @@ proc initDefines*(symbols: StringTableRef) =
     defineSymbol("nimHasWarning" & s)
   for s in HintsToStr:
     defineSymbol("nimHasHint" & s)
+
+  defineSymbol("nimFixedOwned")

compiler/semtypes.nim

@@ -1606,6 +1606,8 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         result = semTypeof(c, n[1], prev)
       elif op.s == "typeof" and n[0].kind == nkSym and n[0].sym.magic == mTypeof:
         result = semTypeOf2(c, n, prev)
+      elif op.s == "owned" and optNimV2 notin c.config.globalOptions and n.len == 2:
+        result = semTypeExpr(c, n[1], prev)
       else:
         if c.inGenericContext > 0 and n.kind == nkCall:
           result = makeTypeFromExpr(c, n.copyTree)

compiler/sigmatch.nim

@@ -1467,10 +1467,14 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
 
   of tyGenericInvocation:
     var x = a.skipGenericAlias
+    var preventHack = false
+    if x.kind == tyOwned and f.sons[0].kind != tyOwned:
+      preventHack = true
+      x = x.lastSon
     # XXX: This is very hacky. It should be moved back into liftTypeParam
     if x.kind in {tyGenericInst, tyArray} and
        c.calleeSym != nil and
-       c.calleeSym.kind in {skProc, skFunc} and c.call != nil:
+       c.calleeSym.kind in {skProc, skFunc} and c.call != nil and not preventHack:
       let inst = prepareMetatypeForSigmatch(c.c, c.bindings, c.call.info, f)
       #echo "inferred ", typeToString(inst), " for ", f
       return typeRel(c, inst, a)

lib/pure/asyncdispatch.nim

@@ -1107,15 +1107,15 @@ else:
       writeList: newSeqOfCap[Callback](InitCallbackListSize)
     )
 
-  proc newDispatcher*(): PDispatcher =
+  proc newDispatcher*(): owned(PDispatcher) =
     new result
     result.selector = newSelector[AsyncData]()
     result.timers.newHeapQueue()
     result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize)
 
-  var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
+  var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher
 
-  proc setGlobalDispatcher*(disp: PDispatcher) =
+  proc setGlobalDispatcher*(disp: owned PDispatcher) =
     if not gDisp.isNil:
       assert gDisp.callbacks.len == 0
     gDisp = disp
@@ -1328,7 +1328,7 @@ else:
     processPendingCallbacks(p, result)
 
   proc recv*(socket: AsyncFD, size: int,
-             flags = {SocketFlag.SafeDisconn}): Future[string] =
+             flags = {SocketFlag.SafeDisconn}): owned(Future[string]) =
     var retFuture = newFuture[string]("recv")
 
     var readBuffer = newString(size)
@@ -1358,7 +1358,7 @@ else:
     return retFuture
 
   proc recvInto*(socket: AsyncFD, buf: pointer, size: int,
-                 flags = {SocketFlag.SafeDisconn}): Future[int] =
+                 flags = {SocketFlag.SafeDisconn}): owned(Future[int]) =
     var retFuture = newFuture[int]("recvInto")
 
     proc cb(sock: AsyncFD): bool =
@@ -1382,7 +1382,7 @@ else:
     return retFuture
 
   proc send*(socket: AsyncFD, buf: pointer, size: int,
-             flags = {SocketFlag.SafeDisconn}): Future[void] =
+             flags = {SocketFlag.SafeDisconn}): owned(Future[void]) =
     var retFuture = newFuture[void]("send")
 
     var written = 0
@@ -1415,7 +1415,7 @@ else:
 
   proc sendTo*(socket: AsyncFD, data: pointer, size: int, saddr: ptr SockAddr,
                saddrLen: SockLen,
-               flags = {SocketFlag.SafeDisconn}): Future[void] =
+               flags = {SocketFlag.SafeDisconn}): owned(Future[void]) =
     ## Sends ``data`` of size ``size`` in bytes to specified destination
     ## (``saddr`` of size ``saddrLen`` in bytes, using socket ``socket``.
     ## The returned future will complete once all data has been sent.
@@ -1445,7 +1445,7 @@ else:
 
   proc recvFromInto*(socket: AsyncFD, data: pointer, size: int,
                      saddr: ptr SockAddr, saddrLen: ptr SockLen,
-                     flags = {SocketFlag.SafeDisconn}): Future[int] =
+                     flags = {SocketFlag.SafeDisconn}): owned(Future[int]) =
     ## Receives a datagram data from ``socket`` into ``data``, which must
     ## be at least of size ``size`` in bytes, address of datagram's sender
     ## will be stored into ``saddr`` and ``saddrLen``. Returned future will
@@ -1468,7 +1468,7 @@ else:
     return retFuture
 
   proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}):
-      Future[tuple[address: string, client: AsyncFD]] =
+      owned(Future[tuple[address: string, client: AsyncFD]]) =
     var retFuture = newFuture[tuple[address: string,
         client: AsyncFD]]("acceptAddr")
     proc cb(sock: AsyncFD): bool =
@@ -1608,7 +1608,7 @@ when defined(windows) or defined(nimdoc):
       saddr.sin_family = uint16(toInt(domain))
       doBind(saddr)
 
-  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
+  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) =
     let retFuture = newFuture[void]("doConnect")
     result = retFuture
 
@@ -1640,7 +1640,7 @@ when defined(windows) or defined(nimdoc):
         GC_unref(ol)
         retFuture.fail(newException(OSError, osErrorMsg(lastError)))
 else:
-  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): Future[void] =
+  proc doConnect(socket: AsyncFD, addrInfo: ptr AddrInfo): owned(Future[void]) =
     let retFuture = newFuture[void]("doConnect")
     result = retFuture
 
@@ -1744,7 +1744,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
   tryNextAddrInfo(nil)
 
 proc dial*(address: string, port: Port,
-           protocol: Protocol = IPPROTO_TCP): Future[AsyncFD] =
+           protocol: Protocol = IPPROTO_TCP): owned(Future[AsyncFD]) =
   ## Establishes connection to the specified ``address``:``port`` pair via the
   ## specified protocol. The procedure iterates through possible
   ## resolutions of the ``address`` until it succeeds, meaning that it
@@ -1759,7 +1759,7 @@ proc dial*(address: string, port: Port,
   asyncAddrInfoLoop(aiList, noFD, protocol)
 
 proc connect*(socket: AsyncFD, address: string, port: Port,
-              domain = Domain.AF_INET): Future[void] =
+              domain = Domain.AF_INET): owned(Future[void]) =
   let retFuture = newFuture[void]("connect")
   result = retFuture
 
@@ -1773,7 +1773,7 @@ proc connect*(socket: AsyncFD, address: string, port: Port,
     socket.SocketHandle.bindToDomain(domain)
   asyncAddrInfoLoop(aiList, socket)
 
-proc sleepAsync*(ms: int | float): Future[void] =
+proc sleepAsync*(ms: int | float): owned(Future[void]) =
   ## Suspends the execution of the current async procedure for the next
   ## ``ms`` milliseconds.
   var retFuture = newFuture[void]("sleepAsync")
@@ -1781,7 +1781,7 @@ proc sleepAsync*(ms: int | float): Future[void] =
   p.timers.push((epochTime() + (ms / 1000), retFuture))
   return retFuture
 
-proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] =
+proc withTimeout*[T](fut: Future[T], timeout: int): owned(Future[bool]) =
   ## Returns a future which will complete once ``fut`` completes or after
   ## ``timeout`` milliseconds has elapsed.
   ##
@@ -1804,7 +1804,7 @@ proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] =
   return retFuture
 
 proc accept*(socket: AsyncFD,
-    flags = {SocketFlag.SafeDisconn}): Future[AsyncFD] =
+    flags = {SocketFlag.SafeDisconn}): owned(Future[AsyncFD]) =
   ## Accepts a new connection. Returns a future containing the client socket
   ## corresponding to that connection.
   ## The future will complete when the connection is successfully accepted.
@@ -1820,7 +1820,7 @@ proc accept*(socket: AsyncFD,
   return retFut
 
 proc send*(socket: AsyncFD, data: string,
-           flags = {SocketFlag.SafeDisconn}): Future[void] =
+           flags = {SocketFlag.SafeDisconn}): owned(Future[void]) =
   ## Sends ``data`` to ``socket``. The returned future will complete once all
   ## data has been sent.
   var retFuture = newFuture[void]("send")
@@ -1843,7 +1843,7 @@ proc send*(socket: AsyncFD, data: string,
 # -- Await Macro
 include asyncmacro
 
-proc readAll*(future: FutureStream[string]): Future[string] {.async.} =
+proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} =
   ## Returns a future that will complete when all the string data from the
   ## specified future stream is retrieved.
   result = ""

lib/pure/asyncfutures.nim

@@ -6,7 +6,7 @@ type
 
   CallbackList = object
     function: CallbackFunc
-    next: ref CallbackList
+    next: owned(ref CallbackList)
 
   FutureBase* = ref object of RootObj ## Untyped future.
     callbacks: CallbackList
@@ -99,7 +99,7 @@ template setupFutureBase(fromProc: string) =
     result.fromProc = fromProc
     currentID.inc()
 
-proc newFuture*[T](fromProc: string = "unspecified"): Future[T] =
+proc newFuture*[T](fromProc: string = "unspecified"): owned(Future[T]) =
   ## Creates a new future.
   ##
   ## Specifying ``fromProc``, which is a string specifying the name of the proc
@@ -107,7 +107,7 @@ proc newFuture*[T](fromProc: string = "unspecified"): Future[T] =
   setupFutureBase(fromProc)
   when isFutureLoggingEnabled: logFutureStart(result)
 
-proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] =
+proc newFutureVar*[T](fromProc = "unspecified"): owned(FutureVar[T]) =
   ## Create a new ``FutureVar``. This Future type is ideally suited for
   ## situations where you want to avoid unnecessary allocations of Futures.
   ##

lib/pure/asyncmacro.nim

@@ -220,8 +220,11 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
 
   let prcName = prc.name.getName
 
-  let returnType = prc.params[0]
+  var returnType = prc.params[0]
   var baseType: NimNode
+  if returnType.kind in nnkCallKinds and returnType[0].eqIdent("owned") and
+      returnType.len == 2:
+    returnType = returnType[1]
   # Verify that the return type is a Future[T]
   if returnType.kind == nnkBracketExpr:
     let fut = repr(returnType[0])

lib/pure/asyncnet.nim

@@ -132,7 +132,7 @@ type
 
 proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET,
     sockType: SockType = SOCK_STREAM,
-    protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket =
+    protocol: Protocol = IPPROTO_TCP, buffered = true): owned(AsyncSocket) =
   ## Creates a new ``AsyncSocket`` based on the supplied params.
   ##
   ## The supplied ``fd``'s non-blocking state will be enabled implicitly.
@@ -152,7 +152,7 @@ proc newAsyncSocket*(fd: AsyncFD, domain: Domain = AF_INET,
     result.currPos = 0
 
 proc newAsyncSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM,
-    protocol: Protocol = IPPROTO_TCP, buffered = true): AsyncSocket =
+    protocol: Protocol = IPPROTO_TCP, buffered = true): owned(AsyncSocket) =
   ## Creates a new asynchronous socket.
   ##
   ## This procedure will also create a brand new file descriptor for
@@ -175,7 +175,7 @@ proc getPeerAddr*(socket: AsyncSocket): (string, Port) =
   getPeerAddr(socket.fd, socket.domain)
 
 proc newAsyncSocket*(domain, sockType, protocol: cint,
-    buffered = true): AsyncSocket =
+    buffered = true): owned(AsyncSocket) =
   ## Creates a new asynchronous socket.
   ##
   ## This procedure will also create a brand new file descriptor for
@@ -216,7 +216,7 @@ when defineSsl:
       await socket.fd.AsyncFd.send(data, flags)
 
   proc appeaseSsl(socket: AsyncSocket, flags: set[SocketFlag],
-                  sslError: cint): Future[bool] {.async.} =
+                  sslError: cint): owned(Future[bool]) {.async.} =
     ## Returns ``true`` if ``socket`` is still connected, otherwise ``false``.
     result = true
     case sslError
@@ -262,7 +262,7 @@ when defineSsl:
             raiseSSLError("Socket has been disconnected")
 
 proc dial*(address: string, port: Port, protocol = IPPROTO_TCP,
-           buffered = true): Future[AsyncSocket] {.async.} =
+           buffered = true): owned(Future[AsyncSocket]) {.async.} =
   ## Establishes connection to the specified ``address``:``port`` pair via the
   ## specified protocol. The procedure iterates through possible
   ## resolutions of the ``address`` until it succeeds, meaning that it
@@ -317,7 +317,7 @@ template readIntoBuf(socket: AsyncSocket,
   size
 
 proc recvInto*(socket: AsyncSocket, buf: pointer, size: int,
-           flags = {SocketFlag.SafeDisconn}): Future[int] {.async.} =
+           flags = {SocketFlag.SafeDisconn}): owned(Future[int]) {.async.} =
   ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``.
   ##
   ## For buffered sockets this function will attempt to read all the requested
@@ -365,7 +365,7 @@ proc recvInto*(socket: AsyncSocket, buf: pointer, size: int,
     result = readInto(buf, size, socket, flags)
 
 proc recv*(socket: AsyncSocket, size: int,
-           flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} =
+           flags = {SocketFlag.SafeDisconn}): owned(Future[string]) {.async.} =
   ## Reads **up to** ``size`` bytes from ``socket``.
   ##
   ## For buffered sockets this function will attempt to read all the requested
@@ -445,7 +445,7 @@ proc send*(socket: AsyncSocket, data: string,
     await send(socket.fd.AsyncFD, data, flags)
 
 proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}):
-      Future[tuple[address: string, client: AsyncSocket]] =
+      owned(Future[tuple[address: string, client: AsyncSocket]]) =
   ## Accepts a new connection. Returns a future containing the client socket
   ## corresponding to that connection and the remote address of the client.
   ## The future will complete when the connection is successfully accepted.
@@ -464,7 +464,7 @@ proc acceptAddr*(socket: AsyncSocket, flags = {SocketFlag.SafeDisconn}):
   return retFuture
 
 proc accept*(socket: AsyncSocket,
-    flags = {SocketFlag.SafeDisconn}): Future[AsyncSocket] =
+    flags = {SocketFlag.SafeDisconn}): owned(Future[AsyncSocket]) =
   ## Accepts a new connection. Returns a future containing the client socket
   ## corresponding to that connection.
   ## The future will complete when the connection is successfully accepted.
@@ -573,7 +573,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
 
 proc recvLine*(socket: AsyncSocket,
     flags = {SocketFlag.SafeDisconn},
-    maxLength = MaxLineLength): Future[string] {.async.} =
+    maxLength = MaxLineLength): owned(Future[string]) {.async.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
   ## a full line is read or an error occurs.
   ##
@@ -632,7 +632,7 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.
 
 when defined(posix):
 
-  proc connectUnix*(socket: AsyncSocket, path: string): Future[void] =
+  proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) =
     ## Binds Unix socket to `path`.
     ## This only works on Unix-style systems: Mac OS X, BSD and Linux
     when not defined(nimdoc):
@@ -663,7 +663,7 @@ when defined(posix):
         else:
           retFuture.fail(newException(OSError, osErrorMsg(lastError)))
 
-  proc bindUnix*(socket: AsyncSocket, path: string)  {.
+  proc bindUnix*(socket: AsyncSocket, path: string) {.
     tags: [ReadIOEffect].} =
     ## Binds Unix socket to `path`.
     ## This only works on Unix-style systems: Mac OS X, BSD and Linux
@@ -675,7 +675,7 @@ when defined(posix):
 
 elif defined(nimdoc):
 
-  proc connectUnix*(socket: AsyncSocket, path: string): Future[void] =
+  proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) =
     ## Binds Unix socket to `path`.
     ## This only works on Unix-style systems: Mac OS X, BSD and Linux
     discard

lib/pure/asyncstreams.nim

@@ -68,7 +68,7 @@ proc write*[T](future: FutureStream[T], value: T): Future[void] =
   if not future.cb.isNil: future.cb()
   result.complete()
 
-proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
+proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) =
   ## Returns a future that will complete when the ``FutureStream`` has data
   ## placed into it. The future will be completed with the oldest
   ## value stored inside the stream. The return value will also determine

lib/pure/ioselects/ioselectors_kqueue.nim

@@ -80,7 +80,7 @@ proc getUnique[T](s: Selector[T]): int {.inline.} =
   if result == -1:
     raiseIOSelectorsError(osLastError())
 
-proc newSelector*[T](): Selector[T] =
+proc newSelector*[T](): owned(Selector[T]) =
   var maxFD = 0.cint
   var size = csize(sizeof(cint))
   var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint]

lib/pure/net.nim

@@ -187,7 +187,7 @@ proc toOSFlags*(socketFlags: set[SocketFlag]): cint =
 
 proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET,
     sockType: SockType = SOCK_STREAM,
-    protocol: Protocol = IPPROTO_TCP, buffered = true): Socket =
+    protocol: Protocol = IPPROTO_TCP, buffered = true): owned(Socket) =
   ## Creates a new socket as specified by the params.
   assert fd != osInvalidSocket
   result = Socket(
@@ -203,7 +203,7 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET,
   when defined(macosx) and not defined(nimdoc):
     setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1)
 
-proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket =
+proc newSocket*(domain, sockType, protocol: cint, buffered = true): owned(Socket) =
   ## Creates a new socket.
   ##
   ## If an error occurs OSError will be raised.
@@ -214,7 +214,7 @@ proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket =
                      buffered)
 
 proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM,
-                protocol: Protocol = IPPROTO_TCP, buffered = true): Socket =
+                protocol: Protocol = IPPROTO_TCP, buffered = true): owned(Socket) =
   ## Creates a new socket.
   ##
   ## If an error occurs OSError will be raised.
@@ -766,7 +766,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {.
     raiseOSError(osLastError())
   freeAddrInfo(aiList)
 
-proc acceptAddr*(server: Socket, client: var Socket, address: var string,
+proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string,
                  flags = {SocketFlag.SafeDisconn}) {.
                  tags: [ReadIOEffect], gcsafe, locks: 0.} =
   ## Blocks until a connection is being made from a client. When a connection
@@ -858,7 +858,7 @@ when false: #defineSsl:
       acceptAddrPlain(AcceptNoClient, AcceptSuccess):
         doHandshake()
 
-proc accept*(server: Socket, client: var Socket,
+proc accept*(server: Socket, client: var owned(Socket),
              flags = {SocketFlag.SafeDisconn}) {.tags: [ReadIOEffect].} =
   ## Equivalent to ``acceptAddr`` but doesn't return the address, only the
   ## socket.
@@ -1509,7 +1509,7 @@ proc `$`*(address: IpAddress): string =
           printedLastGroup = true
 
 proc dial*(address: string, port: Port,
-           protocol = IPPROTO_TCP, buffered = true): Socket
+           protocol = IPPROTO_TCP, buffered = true): owned(Socket)
            {.tags: [ReadIOEffect, WriteIOEffect].} =
   ## Establishes connection to the specified ``address``:``port`` pair via the
   ## specified protocol. The procedure iterates through possible

lib/system.nim

@@ -1689,9 +1689,12 @@ template `isnot`*(x, y: untyped): untyped = not (x is y)
   ##   assert 42 isnot float
   ##   assert @[1, 2] isnot enum
 
-when defined(nimV2) and not defined(nimscript):
+when (defined(nimV2) and not defined(nimscript)) or defined(nimFixedOwned):
   type owned*{.magic: "BuiltinType".}[T] ## type constructor to mark a ref/ptr or a closure as `owned`.
+else:
+  template owned*(t: typeDesc): typedesc = t
 
+when defined(nimV2) and not defined(nimscript):
   proc new*[T](a: var owned(ref T)) {.magic: "New", noSideEffect.}
     ## Creates a new object of type ``T`` and returns a safe (traced)
     ## reference to it in ``a``.
@@ -1716,7 +1719,6 @@ when defined(nimV2) and not defined(nimscript):
   template `<//>`*(t: untyped): untyped = owned(t)
 
 else:
-  template owned*(t: typeDesc): typedesc = t
   template unown*(x: typed): untyped = x
 
   proc new*[T](a: var ref T) {.magic: "New", noSideEffect.}

lib/system/memtracker.nim

@@ -70,7 +70,7 @@ proc addEntry(entry: LogEntry) =
     if interesting:
       gLog.disabled = true
       cprintf("interesting %s:%ld %s\n", entry.file, entry.line, entry.op)
-      let x = cast[proc() {.nimcall, tags: [], gcsafe, locks: 0.}](writeStackTrace)
+      let x = cast[proc() {.nimcall, tags: [], gcsafe, locks: 0, raises: [].}](writeStackTrace)
       x()
       quit 1
       #if gLog.count > high(gLog.data):